summaryrefslogtreecommitdiffstats
path: root/scripts/lib/recipetool/create_npm.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/recipetool/create_npm.py')
-rw-r--r--scripts/lib/recipetool/create_npm.py300
1 files changed, 0 insertions, 300 deletions
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
deleted file mode 100644
index 8c4cdd5234..0000000000
--- a/scripts/lib/recipetool/create_npm.py
+++ /dev/null
@@ -1,300 +0,0 @@
1# Copyright (C) 2016 Intel Corporation
2# Copyright (C) 2020 Savoir-Faire Linux
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Recipe creation tool - npm module support plugin"""
7
8import json
9import logging
10import os
11import re
12import sys
13import tempfile
14import bb
15from bb.fetch2.npm import NpmEnvironment
16from bb.fetch2.npm import npm_package
17from bb.fetch2.npmsw import foreach_dependencies
18from oe.license_finder import match_licenses, find_license_files
19from recipetool.create import RecipeHandler
20from recipetool.create import generate_common_licenses_chksums
21from recipetool.create import split_pkg_licenses
22logger = logging.getLogger('recipetool')
23
24TINFOIL = None
25
26def tinfoil_init(instance):
27 """Initialize tinfoil"""
28 global TINFOIL
29 TINFOIL = instance
30
31class NpmRecipeHandler(RecipeHandler):
32 """Class to handle the npm recipe creation"""
33
34 @staticmethod
35 def _get_registry(lines):
36 """Get the registry value from the 'npm://registry' url"""
37 registry = None
38
39 def _handle_registry(varname, origvalue, op, newlines):
40 nonlocal registry
41 if origvalue.startswith("npm://"):
42 registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
43 return origvalue, None, 0, True
44
45 bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
46
47 return registry
48
49 @staticmethod
50 def _ensure_npm():
51 """Check if the 'npm' command is available in the recipes"""
52 if not TINFOIL.recipes_parsed:
53 TINFOIL.parse_recipes()
54
55 try:
56 d = TINFOIL.parse_recipe("nodejs-native")
57 except bb.providers.NoProvider:
58 bb.error("Nothing provides 'nodejs-native' which is required for the build")
59 bb.note("You will likely need to add a layer that provides nodejs")
60 sys.exit(14)
61
62 bindir = d.getVar("STAGING_BINDIR_NATIVE")
63 npmpath = os.path.join(bindir, "npm")
64
65 if not os.path.exists(npmpath):
66 TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
67
68 if not os.path.exists(npmpath):
69 bb.error("Failed to add 'npm' to sysroot")
70 sys.exit(14)
71
72 return bindir
73
74 @staticmethod
75 def _npm_global_configs(dev):
76 """Get the npm global configuration"""
77 configs = []
78
79 if dev:
80 configs.append(("also", "development"))
81 else:
82 configs.append(("only", "production"))
83
84 configs.append(("save", "false"))
85 configs.append(("package-lock", "false"))
86 configs.append(("shrinkwrap", "false"))
87 return configs
88
89 def _run_npm_install(self, d, srctree, registry, dev):
90 """Run the 'npm install' command without building the addons"""
91 configs = self._npm_global_configs(dev)
92 configs.append(("ignore-scripts", "true"))
93
94 if registry:
95 configs.append(("registry", registry))
96
97 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
98
99 env = NpmEnvironment(d, configs=configs)
100 env.run("npm install", workdir=srctree)
101
102 def _generate_shrinkwrap(self, d, srctree, dev):
103 """Check and generate the 'npm-shrinkwrap.json' file if needed"""
104 configs = self._npm_global_configs(dev)
105
106 env = NpmEnvironment(d, configs=configs)
107 env.run("npm shrinkwrap", workdir=srctree)
108
109 return os.path.join(srctree, "npm-shrinkwrap.json")
110
111 def _handle_licenses(self, srctree, shrinkwrap_file, dev):
112 """Return the extra license files and the list of packages"""
113 licfiles = []
114 packages = {}
115 # Licenses from package.json will point to COMMON_LICENSE_DIR so we need
116 # to associate them explicitely to packages for split_pkg_licenses()
117 fallback_licenses = dict()
118
119 def _find_package_licenses(destdir):
120 """Either find license files, or use package.json metadata"""
121 def _get_licenses_from_package_json(package_json):
122 with open(os.path.join(srctree, package_json), "r") as f:
123 data = json.load(f)
124 if "license" in data:
125 licenses = data["license"].split(" ")
126 licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"]
127 return [], licenses
128 else:
129 return [package_json], None
130
131 basedir = os.path.join(srctree, destdir)
132 licfiles = find_license_files(basedir)
133 if len(licfiles) > 0:
134 return licfiles, None
135 else:
136 # A license wasn't found in the package directory, so we'll use the package.json metadata
137 pkg_json = os.path.join(basedir, "package.json")
138 return _get_licenses_from_package_json(pkg_json)
139
140 def _get_package_licenses(destdir, package):
141 (package_licfiles, package_licenses) = _find_package_licenses(destdir)
142 if package_licfiles:
143 licfiles.extend(package_licfiles)
144 else:
145 fallback_licenses[package] = package_licenses
146
147 # Handle the dependencies
148 def _handle_dependency(name, params, destdir):
149 deptree = destdir.split('node_modules/')
150 suffix = "-".join([npm_package(dep) for dep in deptree])
151 packages["${PN}" + suffix] = destdir
152 _get_package_licenses(destdir, "${PN}" + suffix)
153
154 with open(shrinkwrap_file, "r") as f:
155 shrinkwrap = json.load(f)
156 foreach_dependencies(shrinkwrap, _handle_dependency, dev)
157
158 # Handle the parent package
159 packages["${PN}"] = ""
160 _get_package_licenses(srctree, "${PN}")
161
162 return licfiles, packages, fallback_licenses
163
164 # Handle the peer dependencies
165 def _handle_peer_dependency(self, shrinkwrap_file):
166 """Check if package has peer dependencies and show warning if it is the case"""
167 with open(shrinkwrap_file, "r") as f:
168 shrinkwrap = json.load(f)
169
170 packages = shrinkwrap.get("packages", {})
171 peer_deps = packages.get("", {}).get("peerDependencies", {})
172
173 for peer_dep in peer_deps:
174 peer_dep_yocto_name = npm_package(peer_dep)
175 bb.warn(peer_dep + " is a peer dependencie of the actual package. " +
176 "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
177 % peer_dep_yocto_name)
178
179
180
181 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
182 """Handle the npm recipe creation"""
183
184 if "buildsystem" in handled:
185 return False
186
187 files = RecipeHandler.checkfiles(srctree, ["package.json"])
188
189 if not files:
190 return False
191
192 with open(files[0], "r") as f:
193 data = json.load(f)
194
195 if "name" not in data or "version" not in data:
196 return False
197
198 extravalues["PN"] = npm_package(data["name"])
199 extravalues["PV"] = data["version"]
200
201 if "description" in data:
202 extravalues["SUMMARY"] = data["description"]
203
204 if "homepage" in data:
205 extravalues["HOMEPAGE"] = data["homepage"]
206
207 dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
208 registry = self._get_registry(lines_before)
209
210 bb.note("Checking if npm is available ...")
211 # The native npm is used here (and not the host one) to ensure that the
212 # npm version is high enough to ensure an efficient dependency tree
213 # resolution and avoid issue with the shrinkwrap file format.
214 # Moreover the native npm is mandatory for the build.
215 bindir = self._ensure_npm()
216
217 d = bb.data.createCopy(TINFOIL.config_data)
218 d.prependVar("PATH", bindir + ":")
219 d.setVar("S", srctree)
220
221 bb.note("Generating shrinkwrap file ...")
222 # To generate the shrinkwrap file the dependencies have to be installed
223 # first. During the generation process some files may be updated /
224 # deleted. By default devtool tracks the diffs in the srctree and raises
225 # errors when finishing the recipe if some diffs are found.
226 git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
227 if os.path.exists(git_exclude_file):
228 with open(git_exclude_file, "r+") as f:
229 lines = f.readlines()
230 for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
231 if line not in lines:
232 f.write(line + "\n")
233
234 lock_file = os.path.join(srctree, "package-lock.json")
235 lock_copy = lock_file + ".copy"
236 if os.path.exists(lock_file):
237 bb.utils.copyfile(lock_file, lock_copy)
238
239 self._run_npm_install(d, srctree, registry, dev)
240 shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
241
242 with open(shrinkwrap_file, "r") as f:
243 shrinkwrap = json.load(f)
244
245 if os.path.exists(lock_copy):
246 bb.utils.movefile(lock_copy, lock_file)
247
248 # Add the shrinkwrap file as 'extrafiles'
249 shrinkwrap_copy = shrinkwrap_file + ".copy"
250 bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
251 extravalues.setdefault("extrafiles", {})
252 extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
253
254 url_local = "npmsw://%s" % shrinkwrap_file
255 url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
256
257 if dev:
258 url_local += ";dev=1"
259 url_recipe += ";dev=1"
260
261 # Add the npmsw url in the SRC_URI of the generated recipe
262 def _handle_srcuri(varname, origvalue, op, newlines):
263 """Update the version value and add the 'npmsw://' url"""
264 value = origvalue.replace("version=" + data["version"], "version=${PV}")
265 value = value.replace("version=latest", "version=${PV}")
266 values = [line.strip() for line in value.strip('\n').splitlines()]
267 if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
268 values.append(url_recipe)
269 return values, None, 4, False
270
271 (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
272 lines_before[:] = [line.rstrip('\n') for line in newlines]
273
274 # In order to generate correct licence checksums in the recipe the
275 # dependencies have to be fetched again using the npmsw url
276 bb.note("Fetching npm dependencies ...")
277 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
278 fetcher = bb.fetch2.Fetch([url_local], d)
279 fetcher.download()
280 fetcher.unpack(srctree)
281
282 bb.note("Handling licences ...")
283 (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev)
284 licvalues = match_licenses(licfiles, srctree, d)
285 split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses)
286 fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist]
287 extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d)
288 extravalues["LICENSE"] = fallback_licenses_flat
289
290 classes.append("npm")
291 handled.append("buildsystem")
292
293 # Check if package has peer dependencies and inform the user
294 self._handle_peer_dependency(shrinkwrap_file)
295
296 return True
297
298def register_recipe_handlers(handlers):
299 """Register the npm handler"""
300 handlers.append((NpmRecipeHandler(), 60))