diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/npmsw.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/npmsw.py | 94 |
1 files changed, 76 insertions, 18 deletions
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py index 0c3511d8ab..ff5f8dc755 100644 --- a/bitbake/lib/bb/fetch2/npmsw.py +++ b/bitbake/lib/bb/fetch2/npmsw.py | |||
@@ -24,11 +24,14 @@ import bb | |||
24 | from bb.fetch2 import Fetch | 24 | from bb.fetch2 import Fetch |
25 | from bb.fetch2 import FetchMethod | 25 | from bb.fetch2 import FetchMethod |
26 | from bb.fetch2 import ParameterError | 26 | from bb.fetch2 import ParameterError |
27 | from bb.fetch2 import runfetchcmd | ||
27 | from bb.fetch2 import URI | 28 | from bb.fetch2 import URI |
28 | from bb.fetch2.npm import npm_integrity | 29 | from bb.fetch2.npm import npm_integrity |
29 | from bb.fetch2.npm import npm_localfile | 30 | from bb.fetch2.npm import npm_localfile |
30 | from bb.fetch2.npm import npm_unpack | 31 | from bb.fetch2.npm import npm_unpack |
31 | from bb.utils import is_semver | 32 | from bb.utils import is_semver |
33 | from bb.utils import lockfile | ||
34 | from bb.utils import unlockfile | ||
32 | 35 | ||
33 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): | 36 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): |
34 | """ | 37 | """ |
@@ -38,8 +41,9 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False): | |||
38 | with: | 41 | with: |
39 | name = the package name (string) | 42 | name = the package name (string) |
40 | params = the package parameters (dictionary) | 43 | params = the package parameters (dictionary) |
41 | deptree = the package dependency tree (array of strings) | 44 | destdir = the destination of the package (string) |
42 | """ | 45 | """ |
46 | # For handling old style dependencies entries in shinkwrap files | ||
43 | def _walk_deps(deps, deptree): | 47 | def _walk_deps(deps, deptree): |
44 | for name in deps: | 48 | for name in deps: |
45 | subtree = [*deptree, name] | 49 | subtree = [*deptree, name] |
@@ -49,9 +53,22 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False): | |||
49 | continue | 53 | continue |
50 | elif deps[name].get("bundled", False): | 54 | elif deps[name].get("bundled", False): |
51 | continue | 55 | continue |
52 | callback(name, deps[name], subtree) | 56 | destsubdirs = [os.path.join("node_modules", dep) for dep in subtree] |
53 | 57 | destsuffix = os.path.join(*destsubdirs) | |
54 | _walk_deps(shrinkwrap.get("dependencies", {}), []) | 58 | callback(name, deps[name], destsuffix) |
59 | |||
60 | # packages entry means new style shrinkwrap file, else use dependencies | ||
61 | packages = shrinkwrap.get("packages", None) | ||
62 | if packages is not None: | ||
63 | for package in packages: | ||
64 | if package != "": | ||
65 | name = package.split('node_modules/')[-1] | ||
66 | package_infos = packages.get(package, {}) | ||
67 | if dev == False and package_infos.get("dev", False): | ||
68 | continue | ||
69 | callback(name, package_infos, package) | ||
70 | else: | ||
71 | _walk_deps(shrinkwrap.get("dependencies", {}), []) | ||
55 | 72 | ||
56 | class NpmShrinkWrap(FetchMethod): | 73 | class NpmShrinkWrap(FetchMethod): |
57 | """Class to fetch all package from a shrinkwrap file""" | 74 | """Class to fetch all package from a shrinkwrap file""" |
@@ -72,19 +89,22 @@ class NpmShrinkWrap(FetchMethod): | |||
72 | # Resolve the dependencies | 89 | # Resolve the dependencies |
73 | ud.deps = [] | 90 | ud.deps = [] |
74 | 91 | ||
75 | def _resolve_dependency(name, params, deptree): | 92 | def _resolve_dependency(name, params, destsuffix): |
76 | url = None | 93 | url = None |
77 | localpath = None | 94 | localpath = None |
78 | extrapaths = [] | 95 | extrapaths = [] |
79 | destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] | 96 | unpack = True |
80 | destsuffix = os.path.join(*destsubdirs) | ||
81 | 97 | ||
82 | integrity = params.get("integrity", None) | 98 | integrity = params.get("integrity", None) |
83 | resolved = params.get("resolved", None) | 99 | resolved = params.get("resolved", None) |
84 | version = params.get("version", None) | 100 | version = params.get("version", None) |
85 | 101 | ||
86 | # Handle registry sources | 102 | # Handle registry sources |
87 | if is_semver(version) and resolved and integrity: | 103 | if is_semver(version) and integrity: |
104 | # Handle duplicate dependencies without url | ||
105 | if not resolved: | ||
106 | return | ||
107 | |||
88 | localfile = npm_localfile(name, version) | 108 | localfile = npm_localfile(name, version) |
89 | 109 | ||
90 | uri = URI(resolved) | 110 | uri = URI(resolved) |
@@ -109,7 +129,7 @@ class NpmShrinkWrap(FetchMethod): | |||
109 | 129 | ||
110 | # Handle http tarball sources | 130 | # Handle http tarball sources |
111 | elif version.startswith("http") and integrity: | 131 | elif version.startswith("http") and integrity: |
112 | localfile = os.path.join("npm2", os.path.basename(version)) | 132 | localfile = npm_localfile(os.path.basename(version)) |
113 | 133 | ||
114 | uri = URI(version) | 134 | uri = URI(version) |
115 | uri.params["downloadfilename"] = localfile | 135 | uri.params["downloadfilename"] = localfile |
@@ -121,8 +141,28 @@ class NpmShrinkWrap(FetchMethod): | |||
121 | 141 | ||
122 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) | 142 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) |
123 | 143 | ||
144 | # Handle local tarball and link sources | ||
145 | elif version.startswith("file"): | ||
146 | localpath = version[5:] | ||
147 | if not version.endswith(".tgz"): | ||
148 | unpack = False | ||
149 | |||
124 | # Handle git sources | 150 | # Handle git sources |
125 | elif version.startswith("git"): | 151 | elif version.startswith(("git", "bitbucket","gist")) or ( |
152 | not version.endswith((".tgz", ".tar", ".tar.gz")) | ||
153 | and not version.startswith((".", "@", "/")) | ||
154 | and "/" in version | ||
155 | ): | ||
156 | if version.startswith("github:"): | ||
157 | version = "git+https://github.com/" + version[len("github:"):] | ||
158 | elif version.startswith("gist:"): | ||
159 | version = "git+https://gist.github.com/" + version[len("gist:"):] | ||
160 | elif version.startswith("bitbucket:"): | ||
161 | version = "git+https://bitbucket.org/" + version[len("bitbucket:"):] | ||
162 | elif version.startswith("gitlab:"): | ||
163 | version = "git+https://gitlab.com/" + version[len("gitlab:"):] | ||
164 | elif not version.startswith(("git+","git:")): | ||
165 | version = "git+https://github.com/" + version | ||
126 | regex = re.compile(r""" | 166 | regex = re.compile(r""" |
127 | ^ | 167 | ^ |
128 | git\+ | 168 | git\+ |
@@ -148,15 +188,17 @@ class NpmShrinkWrap(FetchMethod): | |||
148 | 188 | ||
149 | url = str(uri) | 189 | url = str(uri) |
150 | 190 | ||
151 | # local tarball sources and local link sources are unsupported | ||
152 | else: | 191 | else: |
153 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) | 192 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) |
154 | 193 | ||
194 | # name is needed by unpack tracer for module mapping | ||
155 | ud.deps.append({ | 195 | ud.deps.append({ |
196 | "name": name, | ||
156 | "url": url, | 197 | "url": url, |
157 | "localpath": localpath, | 198 | "localpath": localpath, |
158 | "extrapaths": extrapaths, | 199 | "extrapaths": extrapaths, |
159 | "destsuffix": destsuffix, | 200 | "destsuffix": destsuffix, |
201 | "unpack": unpack, | ||
160 | }) | 202 | }) |
161 | 203 | ||
162 | try: | 204 | try: |
@@ -177,17 +219,23 @@ class NpmShrinkWrap(FetchMethod): | |||
177 | # This fetcher resolves multiple URIs from a shrinkwrap file and then | 219 | # This fetcher resolves multiple URIs from a shrinkwrap file and then |
178 | # forwards it to a proxy fetcher. The management of the donestamp file, | 220 | # forwards it to a proxy fetcher. The management of the donestamp file, |
179 | # the lockfile and the checksums are forwarded to the proxy fetcher. | 221 | # the lockfile and the checksums are forwarded to the proxy fetcher. |
180 | ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) | 222 | shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]] |
223 | if shrinkwrap_urls: | ||
224 | ud.proxy = Fetch(shrinkwrap_urls, data) | ||
181 | ud.needdonestamp = False | 225 | ud.needdonestamp = False |
182 | 226 | ||
183 | @staticmethod | 227 | @staticmethod |
184 | def _foreach_proxy_method(ud, handle): | 228 | def _foreach_proxy_method(ud, handle): |
185 | returns = [] | 229 | returns = [] |
186 | for proxy_url in ud.proxy.urls: | 230 | #Check if there are dependencies before try to fetch them |
187 | proxy_ud = ud.proxy.ud[proxy_url] | 231 | if len(ud.deps) > 0: |
188 | proxy_d = ud.proxy.d | 232 | for proxy_url in ud.proxy.urls: |
189 | proxy_ud.setup_localpath(proxy_d) | 233 | proxy_ud = ud.proxy.ud[proxy_url] |
190 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | 234 | proxy_d = ud.proxy.d |
235 | proxy_ud.setup_localpath(proxy_d) | ||
236 | lf = lockfile(proxy_ud.lockfile) | ||
237 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | ||
238 | unlockfile(lf) | ||
191 | return returns | 239 | return returns |
192 | 240 | ||
193 | def verify_donestamp(self, ud, d): | 241 | def verify_donestamp(self, ud, d): |
@@ -224,6 +272,7 @@ class NpmShrinkWrap(FetchMethod): | |||
224 | destsuffix = ud.parm.get("destsuffix") | 272 | destsuffix = ud.parm.get("destsuffix") |
225 | if destsuffix: | 273 | if destsuffix: |
226 | destdir = os.path.join(rootdir, destsuffix) | 274 | destdir = os.path.join(rootdir, destsuffix) |
275 | ud.unpack_tracer.unpack("npm-shrinkwrap", destdir) | ||
227 | 276 | ||
228 | bb.utils.mkdirhier(destdir) | 277 | bb.utils.mkdirhier(destdir) |
229 | bb.utils.copyfile(ud.shrinkwrap_file, | 278 | bb.utils.copyfile(ud.shrinkwrap_file, |
@@ -237,7 +286,16 @@ class NpmShrinkWrap(FetchMethod): | |||
237 | 286 | ||
238 | for dep in manual: | 287 | for dep in manual: |
239 | depdestdir = os.path.join(destdir, dep["destsuffix"]) | 288 | depdestdir = os.path.join(destdir, dep["destsuffix"]) |
240 | npm_unpack(dep["localpath"], depdestdir, d) | 289 | if dep["url"]: |
290 | npm_unpack(dep["localpath"], depdestdir, d) | ||
291 | else: | ||
292 | depsrcdir= os.path.join(destdir, dep["localpath"]) | ||
293 | if dep["unpack"]: | ||
294 | npm_unpack(depsrcdir, depdestdir, d) | ||
295 | else: | ||
296 | bb.utils.mkdirhier(depdestdir) | ||
297 | cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) | ||
298 | runfetchcmd(cmd, d, workdir=depdestdir) | ||
241 | 299 | ||
242 | def clean(self, ud, d): | 300 | def clean(self, ud, d): |
243 | """Clean any existing full or partial download""" | 301 | """Clean any existing full or partial download""" |