From e0bd972ba7eaa5f3b0f5d93bf579cd79290d6601 Mon Sep 17 00:00:00 2001 From: Jean-Marie LEMETAYER Date: Fri, 24 Jan 2020 18:08:13 +0100 Subject: bitbake: fetch2: add the npmsw fetcher This commit adds a new npmsw fetcher that fetches every npm dependencies described in a npm shrinkwrap file: https://docs.npmjs.com/files/shrinkwrap.json.html The main package must be fetched separately: SRC_URI = "npm://registry.url;package=foobar;version=1.0.0 \ npmsw://${THISDIR}/npm-shrinkwrap.json" Since a separation has been created between the package and its dependencies, the package can also be fetched with a non npm fetcher without impacting the general behavior: SRC_URI = "git://github.com/foo/bar.git;protocol=https \ npmsw://${THISDIR}/npm-shrinkwrap.json" (Bitbake rev: f5223be54450bf20e0bfbd53b372a7748a44b475) Signed-off-by: Jean-Marie LEMETAYER Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 2 + bitbake/lib/bb/fetch2/npmsw.py | 255 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 bitbake/lib/bb/fetch2/npmsw.py (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 4fe042739d..1b7f573642 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -1894,6 +1894,7 @@ from . import osc from . import repo from . import clearcase from . import npm +from . import npmsw methods.append(local.Local()) methods.append(wget.Wget()) @@ -1912,3 +1913,4 @@ methods.append(osc.Osc()) methods.append(repo.Repo()) methods.append(clearcase.ClearCase()) methods.append(npm.Npm()) +methods.append(npmsw.NpmShrinkWrap()) diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py new file mode 100644 index 0000000000..0c3511d8ab --- /dev/null +++ b/bitbake/lib/bb/fetch2/npmsw.py @@ -0,0 +1,255 @@ +# Copyright (C) 2020 Savoir-Faire Linux +# +# SPDX-License-Identifier: GPL-2.0-only +# +""" +BitBake 'Fetch' npm shrinkwrap implementation + +npm fetcher support the SRC_URI with format of: +SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..." + +Supported SRC_URI options are: + +- dev + Set to 1 to also install devDependencies. + +- destsuffix + Specifies the directory to use to unpack the dependencies (default: ${S}). +""" + +import json +import os +import re +import bb +from bb.fetch2 import Fetch +from bb.fetch2 import FetchMethod +from bb.fetch2 import ParameterError +from bb.fetch2 import URI +from bb.fetch2.npm import npm_integrity +from bb.fetch2.npm import npm_localfile +from bb.fetch2.npm import npm_unpack +from bb.utils import is_semver + +def foreach_dependencies(shrinkwrap, callback=None, dev=False): + """ + Run a callback for each dependencies of a shrinkwrap file. + The callback is using the format: + callback(name, params, deptree) + with: + name = the package name (string) + params = the package parameters (dictionary) + deptree = the package dependency tree (array of strings) + """ + def _walk_deps(deps, deptree): + for name in deps: + subtree = [*deptree, name] + _walk_deps(deps[name].get("dependencies", {}), subtree) + if callback is not None: + if deps[name].get("dev", False) and not dev: + continue + elif deps[name].get("bundled", False): + continue + callback(name, deps[name], subtree) + + _walk_deps(shrinkwrap.get("dependencies", {}), []) + +class NpmShrinkWrap(FetchMethod): + """Class to fetch all package from a shrinkwrap file""" + + def supports(self, ud, d): + """Check if a given url can be fetched with npmsw""" + return ud.type in ["npmsw"] + + def urldata_init(self, ud, d): + """Init npmsw specific variables within url data""" + + # Get the 'shrinkwrap' parameter + ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0]) + + # Get the 'dev' parameter + ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False) + + # Resolve the dependencies + ud.deps = [] + + def _resolve_dependency(name, params, deptree): + url = None + localpath = None + extrapaths = [] + destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] + destsuffix = os.path.join(*destsubdirs) + + integrity = params.get("integrity", None) + resolved = params.get("resolved", None) + version = params.get("version", None) + + # Handle registry sources + if is_semver(version) and resolved and integrity: + localfile = npm_localfile(name, version) + + uri = URI(resolved) + uri.params["downloadfilename"] = localfile + + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + + url = str(uri) + + localpath = os.path.join(d.getVar("DL_DIR"), localfile) + + # Create a resolve file to mimic the npm fetcher and allow + # re-usability of the downloaded file. + resolvefile = localpath + ".resolved" + + bb.utils.mkdirhier(os.path.dirname(resolvefile)) + with open(resolvefile, "w") as f: + f.write(url) + + extrapaths.append(resolvefile) + + # Handle http tarball sources + elif version.startswith("http") and integrity: + localfile = os.path.join("npm2", os.path.basename(version)) + + uri = URI(version) + uri.params["downloadfilename"] = localfile + + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + + url = str(uri) + + localpath = os.path.join(d.getVar("DL_DIR"), localfile) + + # Handle git sources + elif version.startswith("git"): + regex = re.compile(r""" + ^ + git\+ + (?P[a-z]+) + :// + (?P[^#]+) + \# + (?P[0-9a-f]+) + $ + """, re.VERBOSE) + + match = regex.match(version) + + if not match: + raise ParameterError("Invalid git url: %s" % version, ud.url) + + groups = match.groupdict() + + uri = URI("git://" + str(groups["url"])) + uri.params["protocol"] = str(groups["protocol"]) + uri.params["rev"] = str(groups["rev"]) + uri.params["destsuffix"] = destsuffix + + url = str(uri) + + # local tarball sources and local link sources are unsupported + else: + raise ParameterError("Unsupported dependency: %s" % name, ud.url) + + ud.deps.append({ + "url": url, + "localpath": localpath, + "extrapaths": extrapaths, + "destsuffix": destsuffix, + }) + + try: + with open(ud.shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + except Exception as e: + raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url) + + foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev) + + # Avoid conflicts between the environment data and: + # - the proxy url revision + # - the proxy url checksum + data = bb.data.createCopy(d) + data.delVar("SRCREV") + data.delVarFlags("SRC_URI") + + # This fetcher resolves multiple URIs from a shrinkwrap file and then + # forwards it to a proxy fetcher. The management of the donestamp file, + # the lockfile and the checksums are forwarded to the proxy fetcher. + ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) + ud.needdonestamp = False + + @staticmethod + def _foreach_proxy_method(ud, handle): + returns = [] + for proxy_url in ud.proxy.urls: + proxy_ud = ud.proxy.ud[proxy_url] + proxy_d = ud.proxy.d + proxy_ud.setup_localpath(proxy_d) + returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) + return returns + + def verify_donestamp(self, ud, d): + """Verify the donestamp file""" + def _handle(m, ud, d): + return m.verify_donestamp(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) + + def update_donestamp(self, ud, d): + """Update the donestamp file""" + def _handle(m, ud, d): + m.update_donestamp(ud, d) + self._foreach_proxy_method(ud, _handle) + + def need_update(self, ud, d): + """Force a fetch, even if localpath exists ?""" + def _handle(m, ud, d): + return m.need_update(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) + + def try_mirrors(self, fetch, ud, d, mirrors): + """Try to use a mirror""" + def _handle(m, ud, d): + return m.try_mirrors(fetch, ud, d, mirrors) + return all(self._foreach_proxy_method(ud, _handle)) + + def download(self, ud, d): + """Fetch url""" + ud.proxy.download() + + def unpack(self, ud, rootdir, d): + """Unpack the downloaded dependencies""" + destdir = d.getVar("S") + destsuffix = ud.parm.get("destsuffix") + if destsuffix: + destdir = os.path.join(rootdir, destsuffix) + + bb.utils.mkdirhier(destdir) + bb.utils.copyfile(ud.shrinkwrap_file, + os.path.join(destdir, "npm-shrinkwrap.json")) + + auto = [dep["url"] for dep in ud.deps if not dep["localpath"]] + manual = [dep for dep in ud.deps if dep["localpath"]] + + if auto: + ud.proxy.unpack(destdir, auto) + + for dep in manual: + depdestdir = os.path.join(destdir, dep["destsuffix"]) + npm_unpack(dep["localpath"], depdestdir, d) + + def clean(self, ud, d): + """Clean any existing full or partial download""" + ud.proxy.clean() + + # Clean extra files + for dep in ud.deps: + for path in dep["extrapaths"]: + bb.utils.remove(path) + + def done(self, ud, d): + """Is the download done ?""" + def _handle(m, ud, d): + return m.done(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) -- cgit v1.2.3-54-g00ecf