diff options
-rw-r--r-- | meta/classes/npm.bbclass | 349 |
1 files changed, 274 insertions, 75 deletions
diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 4b1f0a39f0..799b2a3829 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass | |||
@@ -1,94 +1,293 @@ | |||
1 | # Copyright (C) 2020 Savoir-Faire Linux | ||
2 | # | ||
3 | # SPDX-License-Identifier: GPL-2.0-only | ||
4 | # | ||
5 | # This bbclass builds and installs an npm package to the target. The package | ||
6 | # sources files should be fetched in the calling recipe by using the SRC_URI | ||
7 | # variable. The ${S} variable should be updated depending of your fetcher. | ||
8 | # | ||
9 | # Usage: | ||
10 | # SRC_URI = "..." | ||
11 | # inherit npm | ||
12 | # | ||
13 | # Optional variables: | ||
14 | # NPM_ARCH: | ||
15 | # Override the auto generated npm architecture. | ||
16 | # | ||
17 | # NPM_INSTALL_DEV: | ||
18 | # Set to 1 to also install devDependencies. | ||
19 | |||
1 | DEPENDS_prepend = "nodejs-native " | 20 | DEPENDS_prepend = "nodejs-native " |
2 | RDEPENDS_${PN}_prepend = "nodejs " | 21 | RDEPENDS_${PN}_prepend = "nodejs " |
3 | S = "${WORKDIR}/npmpkg" | ||
4 | 22 | ||
5 | def node_pkgname(d): | 23 | NPM_INSTALL_DEV ?= "0" |
6 | bpn = d.getVar('BPN') | 24 | |
7 | if bpn.startswith("node-"): | 25 | def npm_target_arch_map(target_arch): |
8 | return bpn[5:] | 26 | """Maps arch names to npm arch names""" |
9 | return bpn | 27 | import re |
28 | if re.match("p(pc|owerpc)(|64)", target_arch): | ||
29 | return "ppc" | ||
30 | elif re.match("i.86$", target_arch): | ||
31 | return "ia32" | ||
32 | elif re.match("x86_64$", target_arch): | ||
33 | return "x64" | ||
34 | elif re.match("arm64$", target_arch): | ||
35 | return "arm" | ||
36 | return target_arch | ||
37 | |||
38 | NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}" | ||
39 | |||
40 | NPM_PACKAGE = "${WORKDIR}/npm-package" | ||
41 | NPM_CACHE = "${WORKDIR}/npm-cache" | ||
42 | NPM_BUILD = "${WORKDIR}/npm-build" | ||
10 | 43 | ||
11 | NPMPN ?= "${@node_pkgname(d)}" | 44 | def npm_global_configs(d): |
45 | """Get the npm global configuration""" | ||
46 | configs = [] | ||
47 | # Configure the cache directory | ||
48 | configs.append(("cache", d.getVar("NPM_CACHE"))) | ||
49 | return configs | ||
12 | 50 | ||
13 | NPM_INSTALLDIR = "${libdir}/node_modules/${NPMPN}" | 51 | def npm_pack(env, srcdir, workdir): |
52 | """Run 'npm pack' on a specified directory""" | ||
53 | import shlex | ||
54 | cmd = "npm pack %s" % shlex.quote(srcdir) | ||
55 | configs = [("ignore-scripts", "true")] | ||
56 | tarball = env.run(cmd, configs=configs, workdir=workdir).strip("\n") | ||
57 | return os.path.join(workdir, tarball) | ||
14 | 58 | ||
15 | # function maps arch names to npm arch names | 59 | python npm_do_configure() { |
16 | def npm_oe_arch_map(target_arch, d): | 60 | """ |
61 | Step one: configure the npm cache and the main npm package | ||
62 | |||
63 | Every dependencies have been fetched and patched in the source directory. | ||
64 | They have to be packed (this remove unneeded files) and added to the npm | ||
65 | cache to be available for the next step. | ||
66 | |||
67 | The main package and its associated manifest file and shrinkwrap file have | ||
68 | to be configured to take into account these cached dependencies. | ||
69 | """ | ||
70 | import base64 | ||
71 | import copy | ||
72 | import json | ||
17 | import re | 73 | import re |
18 | if re.match('p(pc|owerpc)(|64)', target_arch): return 'ppc' | 74 | import shlex |
19 | elif re.match('i.86$', target_arch): return 'ia32' | 75 | import tempfile |
20 | elif re.match('x86_64$', target_arch): return 'x64' | 76 | from bb.fetch2.npm import NpmEnvironment |
21 | elif re.match('arm64$', target_arch): return 'arm' | 77 | from bb.fetch2.npm import npm_unpack |
22 | return target_arch | 78 | from bb.fetch2.npmsw import foreach_dependencies |
79 | from bb.progress import OutOfProgressHandler | ||
23 | 80 | ||
24 | NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH'), d)}" | 81 | bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True) |
25 | NPM_INSTALL_DEV ?= "0" | 82 | bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True) |
83 | |||
84 | env = NpmEnvironment(d, configs=npm_global_configs(d)) | ||
85 | |||
86 | def _npm_cache_add(tarball): | ||
87 | """Run 'npm cache add' for a specified tarball""" | ||
88 | cmd = "npm cache add %s" % shlex.quote(tarball) | ||
89 | env.run(cmd) | ||
90 | |||
91 | def _npm_integrity(tarball): | ||
92 | """Return the npm integrity of a specified tarball""" | ||
93 | sha512 = bb.utils.sha512_file(tarball) | ||
94 | return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode() | ||
95 | |||
96 | def _npm_version(tarball): | ||
97 | """Return the version of a specified tarball""" | ||
98 | regex = r"-(\d+\.\d+\.\d+(-.*)?(\+.*)?)\.tgz" | ||
99 | return re.search(regex, tarball).group(1) | ||
100 | |||
101 | def _npmsw_dependency_dict(orig, deptree): | ||
102 | """ | ||
103 | Return the sub dictionary in the 'orig' dictionary corresponding to the | ||
104 | 'deptree' dependency tree. This function follows the shrinkwrap file | ||
105 | format. | ||
106 | """ | ||
107 | ptr = orig | ||
108 | for dep in deptree: | ||
109 | if "dependencies" not in ptr: | ||
110 | ptr["dependencies"] = {} | ||
111 | ptr = ptr["dependencies"] | ||
112 | if dep not in ptr: | ||
113 | ptr[dep] = {} | ||
114 | ptr = ptr[dep] | ||
115 | return ptr | ||
116 | |||
117 | # Manage the manifest file and shrinkwrap files | ||
118 | orig_manifest_file = d.expand("${S}/package.json") | ||
119 | orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json") | ||
120 | cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json") | ||
121 | cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json") | ||
122 | |||
123 | with open(orig_manifest_file, "r") as f: | ||
124 | orig_manifest = json.load(f) | ||
125 | |||
126 | cached_manifest = copy.deepcopy(orig_manifest) | ||
127 | cached_manifest.pop("dependencies", None) | ||
128 | cached_manifest.pop("devDependencies", None) | ||
129 | |||
130 | with open(orig_shrinkwrap_file, "r") as f: | ||
131 | orig_shrinkwrap = json.load(f) | ||
132 | |||
133 | cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap) | ||
134 | cached_shrinkwrap.pop("dependencies", None) | ||
135 | |||
136 | # Manage the dependencies | ||
137 | progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$") | ||
138 | progress_total = 1 # also count the main package | ||
139 | progress_done = 0 | ||
140 | |||
141 | def _count_dependency(name, params, deptree): | ||
142 | nonlocal progress_total | ||
143 | progress_total += 1 | ||
144 | |||
145 | def _cache_dependency(name, params, deptree): | ||
146 | destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] | ||
147 | destsuffix = os.path.join(*destsubdirs) | ||
148 | with tempfile.TemporaryDirectory() as tmpdir: | ||
149 | # Add the dependency to the npm cache | ||
150 | destdir = os.path.join(d.getVar("S"), destsuffix) | ||
151 | tarball = npm_pack(env, destdir, tmpdir) | ||
152 | _npm_cache_add(tarball) | ||
153 | # Add its signature to the cached shrinkwrap | ||
154 | dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) | ||
155 | dep["version"] = _npm_version(tarball) | ||
156 | dep["integrity"] = _npm_integrity(tarball) | ||
157 | if params.get("dev", False): | ||
158 | dep["dev"] = True | ||
159 | # Display progress | ||
160 | nonlocal progress_done | ||
161 | progress_done += 1 | ||
162 | progress.write("%d/%d" % (progress_done, progress_total)) | ||
163 | |||
164 | dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) | ||
165 | foreach_dependencies(orig_shrinkwrap, _count_dependency, dev) | ||
166 | foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev) | ||
167 | |||
168 | # Configure the main package | ||
169 | with tempfile.TemporaryDirectory() as tmpdir: | ||
170 | tarball = npm_pack(env, d.getVar("S"), tmpdir) | ||
171 | npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d) | ||
172 | |||
173 | # Configure the cached manifest file and cached shrinkwrap file | ||
174 | def _update_manifest(depkey): | ||
175 | for name in orig_manifest.get(depkey, {}): | ||
176 | version = cached_shrinkwrap["dependencies"][name]["version"] | ||
177 | if depkey not in cached_manifest: | ||
178 | cached_manifest[depkey] = {} | ||
179 | cached_manifest[depkey][name] = version | ||
26 | 180 | ||
27 | npm_do_compile() { | 181 | _update_manifest("dependencies") |
28 | # Copy in any additionally fetched modules | 182 | |
29 | if [ -d ${WORKDIR}/node_modules ] ; then | 183 | if dev: |
30 | cp -a ${WORKDIR}/node_modules ${S}/ | 184 | _update_manifest("devDependencies") |
31 | fi | 185 | |
32 | # changing the home directory to the working directory, the .npmrc will | 186 | with open(cached_manifest_file, "w") as f: |
33 | # be created in this directory | 187 | json.dump(cached_manifest, f, indent=2) |
34 | export HOME=${WORKDIR} | 188 | |
35 | if [ "${NPM_INSTALL_DEV}" = "1" ]; then | 189 | with open(cached_shrinkwrap_file, "w") as f: |
36 | npm config set dev true | 190 | json.dump(cached_shrinkwrap, f, indent=2) |
37 | else | ||
38 | npm config set dev false | ||
39 | fi | ||
40 | npm set cache ${WORKDIR}/npm_cache | ||
41 | # clear cache before every build | ||
42 | npm cache clear --force | ||
43 | # Install pkg into ${S} without going to the registry | ||
44 | if [ "${NPM_INSTALL_DEV}" = "1" ]; then | ||
45 | npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --no-registry install | ||
46 | else | ||
47 | npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry install | ||
48 | fi | ||
49 | } | 191 | } |
50 | 192 | ||
51 | npm_do_install() { | 193 | python npm_do_compile() { |
52 | # changing the home directory to the working directory, the .npmrc will | 194 | """ |
53 | # be created in this directory | 195 | Step two: install the npm package |
54 | export HOME=${WORKDIR} | 196 | |
55 | mkdir -p ${D}${libdir}/node_modules | 197 | Use the configured main package and the cached dependencies to run the |
56 | local NPM_PACKFILE=$(npm pack .) | 198 | installation process. The installation is done in a directory which is |
57 | npm install --prefix ${D}${prefix} -g --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry ${NPM_PACKFILE} | 199 | not the destination directory yet. |
58 | ln -fs node_modules ${D}${libdir}/node | 200 | |
59 | find ${D}${NPM_INSTALLDIR} -type f \( -name "*.a" -o -name "*.d" -o -name "*.o" \) -delete | 201 | A combination of 'npm pack' and 'npm install' is used to ensure that the |
60 | if [ -d ${D}${prefix}/etc ] ; then | 202 | installed files are actual copies instead of symbolic links (which is the |
61 | # This will be empty | 203 | default npm behavior). |
62 | rmdir ${D}${prefix}/etc | 204 | """ |
63 | fi | 205 | import shlex |
206 | import tempfile | ||
207 | from bb.fetch2.npm import NpmEnvironment | ||
208 | |||
209 | bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True) | ||
210 | |||
211 | env = NpmEnvironment(d, configs=npm_global_configs(d)) | ||
212 | |||
213 | dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) | ||
214 | |||
215 | with tempfile.TemporaryDirectory() as tmpdir: | ||
216 | configs = [] | ||
217 | |||
218 | if dev: | ||
219 | configs.append(("also", "development")) | ||
220 | else: | ||
221 | configs.append(("only", "production")) | ||
222 | |||
223 | # Report as many logs as possible for debugging purpose | ||
224 | configs.append(("loglevel", "silly")) | ||
225 | |||
226 | # Configure the installation to be done globally in the build directory | ||
227 | configs.append(("global", "true")) | ||
228 | configs.append(("prefix", d.getVar("NPM_BUILD"))) | ||
229 | |||
230 | # Add node-gyp configuration | ||
231 | configs.append(("arch", d.getVar("NPM_ARCH"))) | ||
232 | configs.append(("release", "true")) | ||
233 | |||
234 | # Pack and install the main package | ||
235 | tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir) | ||
236 | env.run("npm install %s" % shlex.quote(tarball), configs=configs) | ||
64 | } | 237 | } |
65 | 238 | ||
66 | python populate_packages_prepend () { | 239 | npm_do_install() { |
67 | instdir = d.expand('${D}${NPM_INSTALLDIR}') | 240 | # Step three: final install |
68 | extrapackages = oe.package.npm_split_package_dirs(instdir) | 241 | # |
69 | pkgnames = extrapackages.keys() | 242 | # The previous installation have to be filtered to remove some extra files. |
70 | d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames)) | 243 | |
71 | for pkgname in pkgnames: | 244 | rm -rf ${D} |
72 | pkgrelpath, pdata = extrapackages[pkgname] | 245 | |
73 | pkgpath = '${NPM_INSTALLDIR}/' + pkgrelpath | 246 | # Copy the entire lib and bin directories |
74 | # package names can't have underscores but npm packages sometimes use them | 247 | install -d ${D}/${nonarch_libdir} |
75 | oe_pkg_name = pkgname.replace('_', '-') | 248 | cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir} |
76 | expanded_pkgname = d.expand(oe_pkg_name) | 249 | |
77 | d.setVar('FILES_%s' % expanded_pkgname, pkgpath) | 250 | if [ -d "${NPM_BUILD}/bin" ] |
78 | if pdata: | 251 | then |
79 | version = pdata.get('version', None) | 252 | install -d ${D}/${bindir} |
80 | if version: | 253 | cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir} |
81 | d.setVar('PKGV_%s' % expanded_pkgname, version) | 254 | fi |
82 | description = pdata.get('description', None) | 255 | |
83 | if description: | 256 | # If the package (or its dependencies) uses node-gyp to build native addons, |
84 | d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'")) | 257 | # object files, static libraries or other temporary files can be hidden in |
85 | d.appendVar('RDEPENDS_%s' % d.getVar('PN'), ' %s' % ' '.join(pkgnames).replace('_', '-')) | 258 | # the lib directory. To reduce the package size and to avoid QA issues |
259 | # (staticdev with static library files) these files must be removed. | ||
260 | local GYP_REGEX=".*/build/Release/[^/]*.node" | ||
261 | |||
262 | # Remove any node-gyp directory in ${D} to remove temporary build files | ||
263 | for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}") | ||
264 | do | ||
265 | local GYP_D_DIR=${GYP_D_FILE%/Release/*} | ||
266 | |||
267 | rm --recursive --force ${GYP_D_DIR} | ||
268 | done | ||
269 | |||
270 | # Copy only the node-gyp release files | ||
271 | for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}") | ||
272 | do | ||
273 | local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}} | ||
274 | |||
275 | install -d ${GYP_D_FILE%/*} | ||
276 | install -m 755 ${GYP_B_FILE} ${GYP_D_FILE} | ||
277 | done | ||
278 | |||
279 | # Remove the shrinkwrap file which does not need to be packed | ||
280 | rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json | ||
281 | rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json | ||
282 | |||
283 | # node(1) is using /usr/lib/node as default include directory and npm(1) is | ||
284 | # using /usr/lib/node_modules as install directory. Let's make both happy. | ||
285 | ln -fs node_modules ${D}/${nonarch_libdir}/node | ||
86 | } | 286 | } |
87 | 287 | ||
88 | FILES_${PN} += " \ | 288 | FILES_${PN} += " \ |
89 | ${bindir} \ | 289 | ${bindir} \ |
90 | ${libdir}/node \ | 290 | ${nonarch_libdir} \ |
91 | ${NPM_INSTALLDIR} \ | ||
92 | " | 291 | " |
93 | 292 | ||
94 | EXPORT_FUNCTIONS do_compile do_install | 293 | EXPORT_FUNCTIONS do_configure do_compile do_install |