diff options
Diffstat (limited to 'scripts/lib/recipetool/create_npm.py')
| -rw-r--r-- | scripts/lib/recipetool/create_npm.py | 300 |
1 files changed, 0 insertions, 300 deletions
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py deleted file mode 100644 index 8c4cdd5234..0000000000 --- a/scripts/lib/recipetool/create_npm.py +++ /dev/null | |||
| @@ -1,300 +0,0 @@ | |||
| 1 | # Copyright (C) 2016 Intel Corporation | ||
| 2 | # Copyright (C) 2020 Savoir-Faire Linux | ||
| 3 | # | ||
| 4 | # SPDX-License-Identifier: GPL-2.0-only | ||
| 5 | # | ||
| 6 | """Recipe creation tool - npm module support plugin""" | ||
| 7 | |||
| 8 | import json | ||
| 9 | import logging | ||
| 10 | import os | ||
| 11 | import re | ||
| 12 | import sys | ||
| 13 | import tempfile | ||
| 14 | import bb | ||
| 15 | from bb.fetch2.npm import NpmEnvironment | ||
| 16 | from bb.fetch2.npm import npm_package | ||
| 17 | from bb.fetch2.npmsw import foreach_dependencies | ||
| 18 | from oe.license_finder import match_licenses, find_license_files | ||
| 19 | from recipetool.create import RecipeHandler | ||
| 20 | from recipetool.create import generate_common_licenses_chksums | ||
| 21 | from recipetool.create import split_pkg_licenses | ||
| 22 | logger = logging.getLogger('recipetool') | ||
| 23 | |||
| 24 | TINFOIL = None | ||
| 25 | |||
| 26 | def tinfoil_init(instance): | ||
| 27 | """Initialize tinfoil""" | ||
| 28 | global TINFOIL | ||
| 29 | TINFOIL = instance | ||
| 30 | |||
| 31 | class NpmRecipeHandler(RecipeHandler): | ||
| 32 | """Class to handle the npm recipe creation""" | ||
| 33 | |||
| 34 | @staticmethod | ||
| 35 | def _get_registry(lines): | ||
| 36 | """Get the registry value from the 'npm://registry' url""" | ||
| 37 | registry = None | ||
| 38 | |||
| 39 | def _handle_registry(varname, origvalue, op, newlines): | ||
| 40 | nonlocal registry | ||
| 41 | if origvalue.startswith("npm://"): | ||
| 42 | registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0]) | ||
| 43 | return origvalue, None, 0, True | ||
| 44 | |||
| 45 | bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry) | ||
| 46 | |||
| 47 | return registry | ||
| 48 | |||
| 49 | @staticmethod | ||
| 50 | def _ensure_npm(): | ||
| 51 | """Check if the 'npm' command is available in the recipes""" | ||
| 52 | if not TINFOIL.recipes_parsed: | ||
| 53 | TINFOIL.parse_recipes() | ||
| 54 | |||
| 55 | try: | ||
| 56 | d = TINFOIL.parse_recipe("nodejs-native") | ||
| 57 | except bb.providers.NoProvider: | ||
| 58 | bb.error("Nothing provides 'nodejs-native' which is required for the build") | ||
| 59 | bb.note("You will likely need to add a layer that provides nodejs") | ||
| 60 | sys.exit(14) | ||
| 61 | |||
| 62 | bindir = d.getVar("STAGING_BINDIR_NATIVE") | ||
| 63 | npmpath = os.path.join(bindir, "npm") | ||
| 64 | |||
| 65 | if not os.path.exists(npmpath): | ||
| 66 | TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot") | ||
| 67 | |||
| 68 | if not os.path.exists(npmpath): | ||
| 69 | bb.error("Failed to add 'npm' to sysroot") | ||
| 70 | sys.exit(14) | ||
| 71 | |||
| 72 | return bindir | ||
| 73 | |||
| 74 | @staticmethod | ||
| 75 | def _npm_global_configs(dev): | ||
| 76 | """Get the npm global configuration""" | ||
| 77 | configs = [] | ||
| 78 | |||
| 79 | if dev: | ||
| 80 | configs.append(("also", "development")) | ||
| 81 | else: | ||
| 82 | configs.append(("only", "production")) | ||
| 83 | |||
| 84 | configs.append(("save", "false")) | ||
| 85 | configs.append(("package-lock", "false")) | ||
| 86 | configs.append(("shrinkwrap", "false")) | ||
| 87 | return configs | ||
| 88 | |||
| 89 | def _run_npm_install(self, d, srctree, registry, dev): | ||
| 90 | """Run the 'npm install' command without building the addons""" | ||
| 91 | configs = self._npm_global_configs(dev) | ||
| 92 | configs.append(("ignore-scripts", "true")) | ||
| 93 | |||
| 94 | if registry: | ||
| 95 | configs.append(("registry", registry)) | ||
| 96 | |||
| 97 | bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) | ||
| 98 | |||
| 99 | env = NpmEnvironment(d, configs=configs) | ||
| 100 | env.run("npm install", workdir=srctree) | ||
| 101 | |||
| 102 | def _generate_shrinkwrap(self, d, srctree, dev): | ||
| 103 | """Check and generate the 'npm-shrinkwrap.json' file if needed""" | ||
| 104 | configs = self._npm_global_configs(dev) | ||
| 105 | |||
| 106 | env = NpmEnvironment(d, configs=configs) | ||
| 107 | env.run("npm shrinkwrap", workdir=srctree) | ||
| 108 | |||
| 109 | return os.path.join(srctree, "npm-shrinkwrap.json") | ||
| 110 | |||
| 111 | def _handle_licenses(self, srctree, shrinkwrap_file, dev): | ||
| 112 | """Return the extra license files and the list of packages""" | ||
| 113 | licfiles = [] | ||
| 114 | packages = {} | ||
| 115 | # Licenses from package.json will point to COMMON_LICENSE_DIR so we need | ||
| 116 | # to associate them explicitely to packages for split_pkg_licenses() | ||
| 117 | fallback_licenses = dict() | ||
| 118 | |||
| 119 | def _find_package_licenses(destdir): | ||
| 120 | """Either find license files, or use package.json metadata""" | ||
| 121 | def _get_licenses_from_package_json(package_json): | ||
| 122 | with open(os.path.join(srctree, package_json), "r") as f: | ||
| 123 | data = json.load(f) | ||
| 124 | if "license" in data: | ||
| 125 | licenses = data["license"].split(" ") | ||
| 126 | licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"] | ||
| 127 | return [], licenses | ||
| 128 | else: | ||
| 129 | return [package_json], None | ||
| 130 | |||
| 131 | basedir = os.path.join(srctree, destdir) | ||
| 132 | licfiles = find_license_files(basedir) | ||
| 133 | if len(licfiles) > 0: | ||
| 134 | return licfiles, None | ||
| 135 | else: | ||
| 136 | # A license wasn't found in the package directory, so we'll use the package.json metadata | ||
| 137 | pkg_json = os.path.join(basedir, "package.json") | ||
| 138 | return _get_licenses_from_package_json(pkg_json) | ||
| 139 | |||
| 140 | def _get_package_licenses(destdir, package): | ||
| 141 | (package_licfiles, package_licenses) = _find_package_licenses(destdir) | ||
| 142 | if package_licfiles: | ||
| 143 | licfiles.extend(package_licfiles) | ||
| 144 | else: | ||
| 145 | fallback_licenses[package] = package_licenses | ||
| 146 | |||
| 147 | # Handle the dependencies | ||
| 148 | def _handle_dependency(name, params, destdir): | ||
| 149 | deptree = destdir.split('node_modules/') | ||
| 150 | suffix = "-".join([npm_package(dep) for dep in deptree]) | ||
| 151 | packages["${PN}" + suffix] = destdir | ||
| 152 | _get_package_licenses(destdir, "${PN}" + suffix) | ||
| 153 | |||
| 154 | with open(shrinkwrap_file, "r") as f: | ||
| 155 | shrinkwrap = json.load(f) | ||
| 156 | foreach_dependencies(shrinkwrap, _handle_dependency, dev) | ||
| 157 | |||
| 158 | # Handle the parent package | ||
| 159 | packages["${PN}"] = "" | ||
| 160 | _get_package_licenses(srctree, "${PN}") | ||
| 161 | |||
| 162 | return licfiles, packages, fallback_licenses | ||
| 163 | |||
| 164 | # Handle the peer dependencies | ||
| 165 | def _handle_peer_dependency(self, shrinkwrap_file): | ||
| 166 | """Check if package has peer dependencies and show warning if it is the case""" | ||
| 167 | with open(shrinkwrap_file, "r") as f: | ||
| 168 | shrinkwrap = json.load(f) | ||
| 169 | |||
| 170 | packages = shrinkwrap.get("packages", {}) | ||
| 171 | peer_deps = packages.get("", {}).get("peerDependencies", {}) | ||
| 172 | |||
| 173 | for peer_dep in peer_deps: | ||
| 174 | peer_dep_yocto_name = npm_package(peer_dep) | ||
| 175 | bb.warn(peer_dep + " is a peer dependencie of the actual package. " + | ||
| 176 | "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool" | ||
| 177 | % peer_dep_yocto_name) | ||
| 178 | |||
| 179 | |||
| 180 | |||
| 181 | def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): | ||
| 182 | """Handle the npm recipe creation""" | ||
| 183 | |||
| 184 | if "buildsystem" in handled: | ||
| 185 | return False | ||
| 186 | |||
| 187 | files = RecipeHandler.checkfiles(srctree, ["package.json"]) | ||
| 188 | |||
| 189 | if not files: | ||
| 190 | return False | ||
| 191 | |||
| 192 | with open(files[0], "r") as f: | ||
| 193 | data = json.load(f) | ||
| 194 | |||
| 195 | if "name" not in data or "version" not in data: | ||
| 196 | return False | ||
| 197 | |||
| 198 | extravalues["PN"] = npm_package(data["name"]) | ||
| 199 | extravalues["PV"] = data["version"] | ||
| 200 | |||
| 201 | if "description" in data: | ||
| 202 | extravalues["SUMMARY"] = data["description"] | ||
| 203 | |||
| 204 | if "homepage" in data: | ||
| 205 | extravalues["HOMEPAGE"] = data["homepage"] | ||
| 206 | |||
| 207 | dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False) | ||
| 208 | registry = self._get_registry(lines_before) | ||
| 209 | |||
| 210 | bb.note("Checking if npm is available ...") | ||
| 211 | # The native npm is used here (and not the host one) to ensure that the | ||
| 212 | # npm version is high enough to ensure an efficient dependency tree | ||
| 213 | # resolution and avoid issue with the shrinkwrap file format. | ||
| 214 | # Moreover the native npm is mandatory for the build. | ||
| 215 | bindir = self._ensure_npm() | ||
| 216 | |||
| 217 | d = bb.data.createCopy(TINFOIL.config_data) | ||
| 218 | d.prependVar("PATH", bindir + ":") | ||
| 219 | d.setVar("S", srctree) | ||
| 220 | |||
| 221 | bb.note("Generating shrinkwrap file ...") | ||
| 222 | # To generate the shrinkwrap file the dependencies have to be installed | ||
| 223 | # first. During the generation process some files may be updated / | ||
| 224 | # deleted. By default devtool tracks the diffs in the srctree and raises | ||
| 225 | # errors when finishing the recipe if some diffs are found. | ||
| 226 | git_exclude_file = os.path.join(srctree, ".git", "info", "exclude") | ||
| 227 | if os.path.exists(git_exclude_file): | ||
| 228 | with open(git_exclude_file, "r+") as f: | ||
| 229 | lines = f.readlines() | ||
| 230 | for line in ["/node_modules/", "/npm-shrinkwrap.json"]: | ||
| 231 | if line not in lines: | ||
| 232 | f.write(line + "\n") | ||
| 233 | |||
| 234 | lock_file = os.path.join(srctree, "package-lock.json") | ||
| 235 | lock_copy = lock_file + ".copy" | ||
| 236 | if os.path.exists(lock_file): | ||
| 237 | bb.utils.copyfile(lock_file, lock_copy) | ||
| 238 | |||
| 239 | self._run_npm_install(d, srctree, registry, dev) | ||
| 240 | shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev) | ||
| 241 | |||
| 242 | with open(shrinkwrap_file, "r") as f: | ||
| 243 | shrinkwrap = json.load(f) | ||
| 244 | |||
| 245 | if os.path.exists(lock_copy): | ||
| 246 | bb.utils.movefile(lock_copy, lock_file) | ||
| 247 | |||
| 248 | # Add the shrinkwrap file as 'extrafiles' | ||
| 249 | shrinkwrap_copy = shrinkwrap_file + ".copy" | ||
| 250 | bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy) | ||
| 251 | extravalues.setdefault("extrafiles", {}) | ||
| 252 | extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy | ||
| 253 | |||
| 254 | url_local = "npmsw://%s" % shrinkwrap_file | ||
| 255 | url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json" | ||
| 256 | |||
| 257 | if dev: | ||
| 258 | url_local += ";dev=1" | ||
| 259 | url_recipe += ";dev=1" | ||
| 260 | |||
| 261 | # Add the npmsw url in the SRC_URI of the generated recipe | ||
| 262 | def _handle_srcuri(varname, origvalue, op, newlines): | ||
| 263 | """Update the version value and add the 'npmsw://' url""" | ||
| 264 | value = origvalue.replace("version=" + data["version"], "version=${PV}") | ||
| 265 | value = value.replace("version=latest", "version=${PV}") | ||
| 266 | values = [line.strip() for line in value.strip('\n').splitlines()] | ||
| 267 | if "dependencies" in shrinkwrap.get("packages", {}).get("", {}): | ||
| 268 | values.append(url_recipe) | ||
| 269 | return values, None, 4, False | ||
| 270 | |||
| 271 | (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) | ||
| 272 | lines_before[:] = [line.rstrip('\n') for line in newlines] | ||
| 273 | |||
| 274 | # In order to generate correct licence checksums in the recipe the | ||
| 275 | # dependencies have to be fetched again using the npmsw url | ||
| 276 | bb.note("Fetching npm dependencies ...") | ||
| 277 | bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) | ||
| 278 | fetcher = bb.fetch2.Fetch([url_local], d) | ||
| 279 | fetcher.download() | ||
| 280 | fetcher.unpack(srctree) | ||
| 281 | |||
| 282 | bb.note("Handling licences ...") | ||
| 283 | (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev) | ||
| 284 | licvalues = match_licenses(licfiles, srctree, d) | ||
| 285 | split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses) | ||
| 286 | fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist] | ||
| 287 | extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d) | ||
| 288 | extravalues["LICENSE"] = fallback_licenses_flat | ||
| 289 | |||
| 290 | classes.append("npm") | ||
| 291 | handled.append("buildsystem") | ||
| 292 | |||
| 293 | # Check if package has peer dependencies and inform the user | ||
| 294 | self._handle_peer_dependency(shrinkwrap_file) | ||
| 295 | |||
| 296 | return True | ||
| 297 | |||
| 298 | def register_recipe_handlers(handlers): | ||
| 299 | """Register the npm handler""" | ||
| 300 | handlers.append((NpmRecipeHandler(), 60)) | ||
