From ca5b6d66e036fd52a7666480d2009f2a0c1e5216 Mon Sep 17 00:00:00 2001 From: Brendan Le Foll Date: Thu, 25 Feb 2016 15:40:13 +0000 Subject: bitbake: fetch2/npm: Add npm fetcher npm fetcher with support for shrinkwrap files and lockdown files to easily download and install an npm package with strict dependency resolution. The SRC_URI should be in the format of: SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" To add a shrinkwrap and lockdown file use: NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json" NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json" (Bitbake rev: dec75bbc5d075acb322dad8b1c40d6bd518dc9fd) Signed-off-by: Brendan Le Foll Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/npm.py | 226 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 bitbake/lib/bb/fetch2/npm.py (limited to 'bitbake/lib/bb/fetch2/npm.py') diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py new file mode 100644 index 0000000000..54cf76df09 --- /dev/null +++ b/bitbake/lib/bb/fetch2/npm.py @@ -0,0 +1,226 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' NPM implementation + +The NPM fetcher is used to retrieve files from the npmjs repository + +Usage in the recipe: + + SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" + Suported SRC_URI options are: + + - name + - version + + npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};ver=${PV} + The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done + +""" + +import os +import sys +import urllib +import json +import subprocess +import signal +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import ChecksumError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger +from bb.fetch2 import UnpackError +from distutils import spawn + +def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +class Npm(FetchMethod): + + """Class to fetch urls via 'npm'""" + def init(self, d): + pass + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with npm + """ + return ud.type in ['npm'] + + def debug(self, msg): + logger.debug(1, "NpmFetch: %s", msg) + + def clean(self, ud, d): + logger.debug(2, "Calling cleanup %s" % ud.pkgname) + bb.utils.remove(ud.localpath, False) + bb.utils.remove(ud.pkgdatadir, True) + + def urldata_init(self, ud, d): + """ + init NPM specific variable within url data + """ + if 'downloadfilename' in ud.parm: + ud.basename = ud.parm['downloadfilename'] + else: + ud.basename = os.path.basename(ud.path) + + # can't call it ud.name otherwise fetcher base class will start doing sha1stuff + # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps + ud.pkgname = ud.parm.get("name", None) + if not ud.pkgname: + raise ParameterError("NPM fetcher requires a name parameter") + ud.version = ud.parm.get("version", None) + if not ud.version: + raise ParameterError("NPM fetcher requires a version parameter") + ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) + ud.registry = "http://%s" % ud.basename + prefixdir = "npm/%s" % ud.pkgname + ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) + if not os.path.exists(ud.pkgdatadir): + bb.utils.mkdirhiet(ud.pkgdatadir) + ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) + + self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " + self.basecmd += " --directory-prefix=%s " % prefixdir + + def need_update(self, ud, d): + if os.path.exists(ud.localpath): + return False + return True + + def _runwget(self, ud, d, command, quiet): + logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) + bb.fetch2.check_network_access(d, command) + runfetchcmd(command, d, quiet) + + def _unpackdep(self, ud, pkg, data, destdir, dldir, d): + file = data[pkg]['tgz'] + logger.debug(2, "file to extract is %s" % file) + if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): + cmd = 'tar xz --strip 1 --no-same-owner -f %s/%s' % (dldir, file) + else: + bb.fatal("NPM package %s downloaded not a tarball!" % file) + + # Change to subdir before executing command + save_cwd = os.getcwd() + if not os.path.exists(destdir): + os.makedirs(destdir) + os.chdir(destdir) + path = d.getVar('PATH', True) + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (file, os.getcwd())) + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) + + if 'deps' not in data[pkg]: + return + for dep in data[pkg]['deps']: + self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) + + + def unpack(self, ud, destdir, d): + dldir = d.getVar("DL_DIR", True) + depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version) + with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile: + workobj = json.load(datafile) + dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) + + self._unpackdep(ud, ud.pkgname, workobj, "%s/npmpkg" % destdir, dldir, d) + + def _getdependencies(self, pkg, data, version, d, ud): + pkgfullname = pkg + if version: + pkgfullname += "@%s" % version + logger.debug(2, "Calling getdeps on %s" % pkg) + fetchcmd = "npm view %s dist.tarball --registry %s" % (pkgfullname, ud.registry) + output = runfetchcmd(fetchcmd, d, True) + # npm may resolve multiple versions + outputarray = output.strip().splitlines() + # we just take the latest version npm resolved + #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) + outputurl = outputarray[len(outputarray)-1].rstrip() + if (len(outputarray) > 1): + # remove the preceding version/name from npm output and then the + # first and last quotes + outputurl = outputurl.split(" ")[1][1:-1] + data[pkg] = {} + data[pkg]['tgz'] = os.path.basename(outputurl) + self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False) + #fetchcmd = "npm view %s@%s dependencies --json" % (pkg, version) + fetchcmd = "npm view %s dependencies --json --registry %s" % (pkgfullname, ud.registry) + output = runfetchcmd(fetchcmd, d, True) + try: + depsfound = json.loads(output) + except: + # just assume there is no deps to be loaded here + return + data[pkg]['deps'] = {} + for dep, version in depsfound.iteritems(): + self._getdependencies(dep, data[pkg]['deps'], version, d, ud) + + def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest): + logger.debug(2, "NPM shrinkwrap file is %s" % data) + outputurl = "invalid" + if ('resolved' not in data): + # will be the case for ${PN} + fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) + logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) + outputurl = runfetchcmd(fetchcmd, d, True) + else: + outputurl = data['resolved'] + self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False) + manifest[pkg] = {} + manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() + manifest[pkg]['deps'] = {} + + if pkg in lockdown: + sha1_expected = lockdown[pkg][version] + sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) + if sha1_expected != sha1_data: + msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) + raise ChecksumError('Checksum mismatch!%s' % msg) + else: + logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) + + if 'dependencies' in data: + for obj in data['dependencies']: + logger.debug(2, "Found dep is %s" % str(obj)) + self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps']) + + def download(self, ud, d): + """Fetch url""" + jsondepobj = {} + shrinkobj = {} + lockdown = {} + + shwrf = d.getVar('NPM_SHRINKWRAP', True) + logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) + try: + with open(shwrf) as datafile: + shrinkobj = json.load(datafile) + except: + logger.warn('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) + lckdf = d.getVar('NPM_LOCKDOWN', True) + logger.debug(2, "NPM lockdown file is %s" % lckdf) + try: + with open(lckdf) as datafile: + lockdown = json.load(datafile) + except: + logger.warn('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) + + if ('name' not in shrinkobj): + self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) + else: + self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) + + with open(ud.localpath, 'w') as outfile: + json.dump(jsondepobj, outfile) -- cgit v1.2.3-54-g00ecf