summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
commit8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch)
treeefdc32587159d0050a69009bdf2330a531727d95 /bitbake/lib/bb/fetch2
parentd412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff)
downloadpoky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz
The poky repository master branch is no longer being updated.
You can either: a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs b) use the new bitbake-setup You can find information about either approach in our documentation: https://docs.yoctoproject.org/ Note that "poky" the distro setting is still available in meta-yocto as before and we continue to use and maintain that. Long live Poky! Some further information on the background of this change can be found in: https://lists.openembedded.org/g/openembedded-architecture/message/2179 Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r--bitbake/lib/bb/fetch2/README57
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py2144
-rw-r--r--bitbake/lib/bb/fetch2/az.py98
-rw-r--r--bitbake/lib/bb/fetch2/bzr.py128
-rw-r--r--bitbake/lib/bb/fetch2/clearcase.py245
-rw-r--r--bitbake/lib/bb/fetch2/crate.py150
-rw-r--r--bitbake/lib/bb/fetch2/cvs.py157
-rw-r--r--bitbake/lib/bb/fetch2/gcp.py102
-rw-r--r--bitbake/lib/bb/fetch2/git.py1010
-rw-r--r--bitbake/lib/bb/fetch2/gitannex.py77
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py278
-rw-r--r--bitbake/lib/bb/fetch2/gomod.py273
-rw-r--r--bitbake/lib/bb/fetch2/hg.py264
-rw-r--r--bitbake/lib/bb/fetch2/local.py91
-rw-r--r--bitbake/lib/bb/fetch2/npm.py331
-rw-r--r--bitbake/lib/bb/fetch2/npmsw.py291
-rw-r--r--bitbake/lib/bb/fetch2/osc.py165
-rw-r--r--bitbake/lib/bb/fetch2/perforce.py267
-rw-r--r--bitbake/lib/bb/fetch2/repo.py87
-rw-r--r--bitbake/lib/bb/fetch2/s3.py124
-rw-r--r--bitbake/lib/bb/fetch2/sftp.py112
-rw-r--r--bitbake/lib/bb/fetch2/ssh.py154
-rw-r--r--bitbake/lib/bb/fetch2/svn.py215
-rw-r--r--bitbake/lib/bb/fetch2/wget.py693
24 files changed, 0 insertions, 7513 deletions
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README
deleted file mode 100644
index 67b787ef47..0000000000
--- a/bitbake/lib/bb/fetch2/README
+++ /dev/null
@@ -1,57 +0,0 @@
1There are expectations of users of the fetcher code. This file attempts to document
2some of the constraints that are present. Some are obvious, some are less so. It is
3documented in the context of how OE uses it but the API calls are generic.
4
5a) network access for sources is only expected to happen in the do_fetch task.
6 This is not enforced or tested but is required so that we can:
7
8 i) audit the sources used (i.e. for license/manifest reasons)
9 ii) support offline builds with a suitable cache
10 iii) allow work to continue even with downtime upstream
11 iv) allow for changes upstream in incompatible ways
12 v) allow rebuilding of the software in X years time
13
14b) network access is not expected in do_unpack task.
15
16c) you can take DL_DIR and use it as a mirror for offline builds.
17
18d) access to the network is only made when explicitly configured in recipes
19 (e.g. use of AUTOREV, or use of git tags which change revision).
20
21e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it
22 will match in future exactly in a clean build with a new DL_DIR).
23 One specific pain point example are git tags. They can be replaced and change
24 so the git fetcher has to resolve them with the network. We use git revisions
25 where possible to avoid this and ensure determinism.
26
27f) network access is expected to work with the standard linux proxy variables
28 so that access behind firewalls works (the fetcher sets these in the
29 environment but only in the do_fetch tasks).
30
31g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV
32 git recipe might be ok but you can't expect to checkout a git tree.
33
34h) we need to provide revision information during parsing such that a version
35 for the recipe can be constructed.
36
37i) versions are expected to be able to increase in a way which sorts allowing
38 package feeds to operate (see PR server required for git revisions to sort).
39
40j) API to query for possible version upgrades of a url is highly desireable to
41 allow our automated upgrage code to function (it is implied this does always
42 have network access).
43
44k) Where fixes or changes to behaviour in the fetcher are made, we ask that
45 test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do
46 have fairly extensive test coverage of the fetcher as it is the only way
47 to track all of its corner cases, it still doesn't give entire coverage
48 though sadly.
49
50l) If using tools during parse time, they will have to be in ASSUME_PROVIDED
51 in OE's context as we can't build git-native, then parse a recipe and use
52 git ls-remote.
53
54Not all fetchers support all features, autorev is optional and doesn't make
55sense for some. Upgrade detection means different things in different contexts
56too.
57
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
deleted file mode 100644
index 0ad987c596..0000000000
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ /dev/null
@@ -1,2144 +0,0 @@
1"""
2BitBake 'Fetch' implementations
3
4Classes for obtaining upstream sources for the
5BitBake build tools.
6"""
7
8# Copyright (C) 2003, 2004 Chris Larson
9# Copyright (C) 2012 Intel Corporation
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import os, re
16import signal
17import logging
18import urllib.request, urllib.parse, urllib.error
19if 'git' not in urllib.parse.uses_netloc:
20 urllib.parse.uses_netloc.append('git')
21import operator
22import collections
23import subprocess
24import pickle
25import errno
26import bb.utils
27import bb.checksum
28import bb.process
29import bb.event
30
31__version__ = "2"
32_checksum_cache = bb.checksum.FileChecksumCache()
33_revisions_cache = bb.checksum.RevisionsCache()
34
35logger = logging.getLogger("BitBake.Fetcher")
36
37CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ]
38SHOWN_CHECKSUM_LIST = ["sha256"]
39
40class BBFetchException(Exception):
41 """Class all fetch exceptions inherit from"""
42 def __init__(self, message):
43 self.msg = message
44 Exception.__init__(self, message)
45
46 def __str__(self):
47 return self.msg
48
49class UntrustedUrl(BBFetchException):
50 """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS"""
51 def __init__(self, url, message=''):
52 if message:
53 msg = message
54 else:
55 msg = "The URL: '%s' is not trusted and cannot be used" % url
56 self.url = url
57 BBFetchException.__init__(self, msg)
58 self.args = (url,)
59
60class MalformedUrl(BBFetchException):
61 """Exception raised when encountering an invalid url"""
62 def __init__(self, url, message=''):
63 if message:
64 msg = message
65 else:
66 msg = "The URL: '%s' is invalid and cannot be interpreted" % url
67 self.url = url
68 BBFetchException.__init__(self, msg)
69 self.args = (url,)
70
71class FetchError(BBFetchException):
72 """General fetcher exception when something happens incorrectly"""
73 def __init__(self, message, url = None):
74 if url:
75 msg = "Fetcher failure for URL: '%s'. %s" % (url, message)
76 else:
77 msg = "Fetcher failure: %s" % message
78 self.url = url
79 BBFetchException.__init__(self, msg)
80 self.args = (message, url)
81
82class ChecksumError(FetchError):
83 """Exception when mismatched checksum encountered"""
84 def __init__(self, message, url = None, checksum = None):
85 self.checksum = checksum
86 FetchError.__init__(self, message, url)
87
88class NoChecksumError(FetchError):
89 """Exception when no checksum is specified, but BB_STRICT_CHECKSUM is set"""
90
91class UnpackError(BBFetchException):
92 """General fetcher exception when something happens incorrectly when unpacking"""
93 def __init__(self, message, url):
94 msg = "Unpack failure for URL: '%s'. %s" % (url, message)
95 self.url = url
96 BBFetchException.__init__(self, msg)
97 self.args = (message, url)
98
99class NoMethodError(BBFetchException):
100 """Exception raised when there is no method to obtain a supplied url or set of urls"""
101 def __init__(self, url):
102 msg = "Could not find a fetcher which supports the URL: '%s'" % url
103 self.url = url
104 BBFetchException.__init__(self, msg)
105 self.args = (url,)
106
107class MissingParameterError(BBFetchException):
108 """Exception raised when a fetch method is missing a critical parameter in the url"""
109 def __init__(self, missing, url):
110 msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing)
111 self.url = url
112 self.missing = missing
113 BBFetchException.__init__(self, msg)
114 self.args = (missing, url)
115
116class ParameterError(BBFetchException):
117 """Exception raised when a url cannot be processed due to invalid parameters."""
118 def __init__(self, message, url):
119 msg = "URL: '%s' has invalid parameters. %s" % (url, message)
120 self.url = url
121 BBFetchException.__init__(self, msg)
122 self.args = (message, url)
123
124class NetworkAccess(BBFetchException):
125 """Exception raised when network access is disabled but it is required."""
126 def __init__(self, url, cmd):
127 msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url)
128 self.url = url
129 self.cmd = cmd
130 BBFetchException.__init__(self, msg)
131 self.args = (url, cmd)
132
133class NonLocalMethod(Exception):
134 def __init__(self):
135 Exception.__init__(self)
136
137class MissingChecksumEvent(bb.event.Event):
138 def __init__(self, url, **checksums):
139 self.url = url
140 self.checksums = checksums
141 bb.event.Event.__init__(self)
142
143
144class URI(object):
145 """
146 A class representing a generic URI, with methods for
147 accessing the URI components, and stringifies to the
148 URI.
149
150 It is constructed by calling it with a URI, or setting
151 the attributes manually:
152
153 uri = URI("http://example.com/")
154
155 uri = URI()
156 uri.scheme = 'http'
157 uri.hostname = 'example.com'
158 uri.path = '/'
159
160 It has the following attributes:
161
162 * scheme (read/write)
163 * userinfo (authentication information) (read/write)
164 * username (read/write)
165 * password (read/write)
166
167 Note, password is deprecated as of RFC 3986.
168
169 * hostname (read/write)
170 * port (read/write)
171 * hostport (read only)
172 "hostname:port", if both are set, otherwise just "hostname"
173 * path (read/write)
174 * path_quoted (read/write)
175 A URI quoted version of path
176 * params (dict) (read/write)
177 * query (dict) (read/write)
178 * relative (bool) (read only)
179 True if this is a "relative URI", (e.g. file:foo.diff)
180
181 It stringifies to the URI itself.
182
183 Some notes about relative URIs: while it's specified that
184 a URI beginning with <scheme>:// should either be directly
185 followed by a hostname or a /, the old URI handling of the
186 fetch2 library did not conform to this. Therefore, this URI
187 class has some kludges to make sure that URIs are parsed in
188 a way comforming to bitbake's current usage. This URI class
189 supports the following:
190
191 file:relative/path.diff (IETF compliant)
192 git:relative/path.git (IETF compliant)
193 git:///absolute/path.git (IETF compliant)
194 file:///absolute/path.diff (IETF compliant)
195
196 file://relative/path.diff (not IETF compliant)
197
198 But it does not support the following:
199
200 file://hostname/absolute/path.diff (would be IETF compliant)
201
202 Note that the last case only applies to a list of
203 explicitly allowed schemes (currently only file://), that requires
204 its URIs to not have a network location.
205 """
206
207 _relative_schemes = ['file', 'git']
208 _netloc_forbidden = ['file']
209
210 def __init__(self, uri=None):
211 self.scheme = ''
212 self.userinfo = ''
213 self.hostname = ''
214 self.port = None
215 self._path = ''
216 self.params = {}
217 self.query = {}
218 self.relative = False
219
220 if not uri:
221 return
222
223 # We hijack the URL parameters, since the way bitbake uses
224 # them are not quite RFC compliant.
225 uri, param_str = (uri.split(";", 1) + [None])[:2]
226
227 urlp = urllib.parse.urlparse(uri)
228 self.scheme = urlp.scheme
229
230 reparse = 0
231
232 # Coerce urlparse to make URI scheme use netloc
233 if not self.scheme in urllib.parse.uses_netloc:
234 urllib.parse.uses_params.append(self.scheme)
235 reparse = 1
236
237 # Make urlparse happy(/ier) by converting local resources
238 # to RFC compliant URL format. E.g.:
239 # file://foo.diff -> file:foo.diff
240 if urlp.scheme in self._netloc_forbidden:
241 uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1)
242 reparse = 1
243
244 if reparse:
245 urlp = urllib.parse.urlparse(uri)
246
247 # Identify if the URI is relative or not
248 if urlp.scheme in self._relative_schemes and \
249 re.compile(r"^\w+:(?!//)").match(uri):
250 self.relative = True
251
252 if not self.relative:
253 self.hostname = urlp.hostname or ''
254 self.port = urlp.port
255
256 self.userinfo += urlp.username or ''
257
258 if urlp.password:
259 self.userinfo += ':%s' % urlp.password
260
261 self.path = urllib.parse.unquote(urlp.path)
262
263 if param_str:
264 self.params = self._param_str_split(param_str, ";")
265 if urlp.query:
266 self.query = self._param_str_split(urlp.query, "&")
267
268 def __str__(self):
269 userinfo = self.userinfo
270 if userinfo:
271 userinfo += '@'
272
273 return "%s:%s%s%s%s%s%s" % (
274 self.scheme,
275 '' if self.relative else '//',
276 userinfo,
277 self.hostport,
278 self.path_quoted,
279 self._query_str(),
280 self._param_str())
281
282 def _param_str(self):
283 return (
284 ''.join([';', self._param_str_join(self.params, ";")])
285 if self.params else '')
286
287 def _query_str(self):
288 return (
289 ''.join(['?', self._param_str_join(self.query, "&")])
290 if self.query else '')
291
292 def _param_str_split(self, string, elmdelim, kvdelim="="):
293 ret = collections.OrderedDict()
294 for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
295 ret[k] = v
296 return ret
297
298 def _param_str_join(self, dict_, elmdelim, kvdelim="="):
299 return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
300
301 @property
302 def hostport(self):
303 if not self.port:
304 return self.hostname
305 return "%s:%d" % (self.hostname, self.port)
306
307 @property
308 def path_quoted(self):
309 return urllib.parse.quote(self.path)
310
311 @path_quoted.setter
312 def path_quoted(self, path):
313 self.path = urllib.parse.unquote(path)
314
315 @property
316 def path(self):
317 return self._path
318
319 @path.setter
320 def path(self, path):
321 self._path = path
322
323 if not path or re.compile("^/").match(path):
324 self.relative = False
325 else:
326 self.relative = True
327
328 @property
329 def username(self):
330 if self.userinfo:
331 return (self.userinfo.split(":", 1))[0]
332 return ''
333
334 @username.setter
335 def username(self, username):
336 password = self.password
337 self.userinfo = username
338 if password:
339 self.userinfo += ":%s" % password
340
341 @property
342 def password(self):
343 if self.userinfo and ":" in self.userinfo:
344 return (self.userinfo.split(":", 1))[1]
345 return ''
346
347 @password.setter
348 def password(self, password):
349 self.userinfo = "%s:%s" % (self.username, password)
350
351def decodeurl(url):
352 """Decodes an URL into the tokens (scheme, network location, path,
353 user, password, parameters).
354 """
355
356 uri = URI(url)
357 path = uri.path if uri.path else "/"
358 return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params
359
360def decodemirrorurl(url):
361 """Decodes a mirror URL into the tokens (scheme, network location, path,
362 user, password, parameters).
363 """
364 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
365 if not m:
366 raise MalformedUrl(url)
367
368 type = m.group('type')
369 location = m.group('location')
370 if not location:
371 raise MalformedUrl(url)
372 user = m.group('user')
373 parm = m.group('parm')
374
375 locidx = location.find('/')
376 if locidx != -1 and type.lower() != 'file':
377 host = location[:locidx]
378 path = location[locidx:]
379 elif type.lower() == 'file':
380 host = ""
381 path = location
382 if user:
383 path = user + '@' + path
384 user = ""
385 else:
386 host = location
387 path = "/"
388 if user:
389 m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user)
390 if m:
391 user = m.group('user')
392 pswd = m.group('pswd')
393 else:
394 user = ''
395 pswd = ''
396
397 p = collections.OrderedDict()
398 if parm:
399 for s in parm.split(';'):
400 if s:
401 if not '=' in s:
402 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
403 s1, s2 = s.split('=', 1)
404 p[s1] = s2
405
406 return type, host, urllib.parse.unquote(path), user, pswd, p
407
408def encodeurl(decoded):
409 """Encodes a URL from tokens (scheme, network location, path,
410 user, password, parameters).
411 """
412
413 type, host, path, user, pswd, p = decoded
414
415 if not type:
416 raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
417 uri = URI()
418 uri.scheme = type
419 if user and type != "file":
420 uri.username = user
421 if pswd:
422 uri.password = pswd
423 if host and type != "file":
424 uri.hostname = host
425 if path:
426 # Standardise path to ensure comparisons work
427 while '//' in path:
428 path = path.replace("//", "/")
429 uri.path = path
430 if type == "file":
431 # Use old not IETF compliant style
432 uri.relative = False
433 if p:
434 uri.params = p
435
436 return str(uri)
437
438def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
439 if not ud.url or not uri_find or not uri_replace:
440 logger.error("uri_replace: passed an undefined value, not replacing")
441 return None
442 uri_decoded = list(decodemirrorurl(ud.url))
443 uri_find_decoded = list(decodemirrorurl(uri_find))
444 uri_replace_decoded = list(decodemirrorurl(uri_replace))
445 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
446 result_decoded = ['', '', '', '', '', {}]
447 # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params
448 for loc, i in enumerate(uri_find_decoded):
449 result_decoded[loc] = uri_decoded[loc]
450 regexp = i
451 if loc == 0 and regexp and not regexp.endswith("$"):
452 # Leaving the type unanchored can mean "https" matching "file" can become "files"
453 # which is clearly undesirable.
454 regexp += "$"
455 if loc == 5:
456 # Handle URL parameters
457 if i:
458 # Any specified URL parameters must match
459 for k in uri_find_decoded[loc]:
460 if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
461 return None
462 # Overwrite any specified replacement parameters
463 for k in uri_replace_decoded[loc]:
464 for l in replacements:
465 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
466 result_decoded[loc][k] = uri_replace_decoded[loc][k]
467 elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
468 # User/password in the replacement is just a straight replacement
469 result_decoded[loc] = uri_replace_decoded[loc]
470 elif (re.match(regexp, uri_decoded[loc])):
471 if not uri_replace_decoded[loc]:
472 result_decoded[loc] = ""
473 else:
474 for k in replacements:
475 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
476 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
477 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
478 if loc == 2:
479 # Handle path manipulations
480 basename = None
481 if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball:
482 # If the source and destination url types differ, must be a mirrortarball mapping
483 basename = os.path.basename(mirrortarball)
484 # Kill parameters, they make no sense for mirror tarballs
485 uri_decoded[5] = {}
486 uri_find_decoded[5] = {}
487 elif ud.localpath and ud.method.supports_checksum(ud):
488 basename = os.path.basename(ud.localpath)
489 if basename:
490 uri_basename = os.path.basename(uri_decoded[loc])
491 # Prefix with a slash as a sentinel in case
492 # result_decoded[loc] does not contain one.
493 path = "/" + result_decoded[loc]
494 if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename):
495 result_decoded[loc] = path[1:-len(uri_basename)] + basename
496 elif not path.endswith("/" + basename):
497 result_decoded[loc] = os.path.join(path[1:], basename)
498 else:
499 return None
500 result = encodeurl(result_decoded)
501 if result == ud.url:
502 return None
503 logger.debug2("For url %s returning %s" % (ud.url, result))
504 return result
505
506methods = []
507urldata_cache = {}
508saved_headrevs = {}
509
510def fetcher_init(d, servercontext=True):
511 """
512 Called to initialize the fetchers once the configuration data is known.
513 Calls before this must not hit the cache.
514 """
515
516 _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
517 _revisions_cache.init_cache(d.getVar("BB_CACHEDIR"))
518
519 if not servercontext:
520 return
521
522 try:
523 # fetcher_init is called multiple times, so make sure we only save the
524 # revs the first time it is called.
525 if not bb.fetch2.saved_headrevs:
526 bb.fetch2.saved_headrevs = _revisions_cache.get_revs()
527 except:
528 pass
529
530 # When to drop SCM head revisions controlled by user policy
531 srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
532 if srcrev_policy == "cache":
533 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
534 elif srcrev_policy == "clear":
535 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
536 _revisions_cache.clear_cache()
537 else:
538 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
539
540
541 for m in methods:
542 if hasattr(m, "init"):
543 m.init(d)
544
545def fetcher_parse_save():
546 _checksum_cache.save_extras()
547 _revisions_cache.save_extras()
548
549def fetcher_parse_done():
550 _checksum_cache.save_merge()
551 _revisions_cache.save_merge()
552
553def fetcher_compare_revisions(d):
554 """
555 Compare the revisions in the persistent cache with the saved values from
556 when bitbake was started and return true if they have changed.
557 """
558
559 headrevs = _revisions_cache.get_revs()
560 return headrevs != bb.fetch2.saved_headrevs
561
562def mirror_from_string(data):
563 mirrors = (data or "").replace('\\n',' ').split()
564 # Split into pairs
565 if len(mirrors) % 2 != 0:
566 bb.warn('Invalid mirror data %s, should have paired members.' % data)
567 return list(zip(*[iter(mirrors)]*2))
568
569def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
570 """
571 verify the MD5 and SHA256 checksum for downloaded src
572
573 Raises a FetchError if one or both of the SRC_URI checksums do not match
574 the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no
575 checksums specified.
576
577 Returns a dict of checksums that can be stored in a done stamp file and
578 passed in as precomputed parameter in a later call to avoid re-computing
579 the checksums from the file. This allows verifying the checksums of the
580 file against those in the recipe each time, rather than only after
581 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
582 """
583 if ud.ignore_checksums or not ud.method.supports_checksum(ud):
584 return {}
585
586 if localpath is None:
587 localpath = ud.localpath
588
589 def compute_checksum_info(checksum_id):
590 checksum_name = getattr(ud, "%s_name" % checksum_id)
591
592 if checksum_id in precomputed:
593 checksum_data = precomputed[checksum_id]
594 else:
595 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
596
597 checksum_expected = getattr(ud, "%s_expected" % checksum_id)
598
599 if checksum_expected == '':
600 checksum_expected = None
601
602 return {
603 "id": checksum_id,
604 "name": checksum_name,
605 "data": checksum_data,
606 "expected": checksum_expected
607 }
608
609 checksum_infos = []
610 for checksum_id in CHECKSUM_LIST:
611 checksum_infos.append(compute_checksum_info(checksum_id))
612
613 checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos}
614 checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos}
615
616 for ci in checksum_infos:
617 if ci["id"] in SHOWN_CHECKSUM_LIST:
618 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
619
620 # If no checksum has been provided
621 if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
622 messages = []
623 strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
624
625 # If strict checking enabled and neither sum defined, raise error
626 if strict == "1":
627 raise NoChecksumError("\n".join(checksum_lines))
628
629 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
630
631 if strict == "ignore":
632 return checksum_dict
633
634 # Log missing sums so user can more easily add them
635 messages.append("Missing checksum for '%s', consider adding at " \
636 "least one to the recipe:" % ud.localpath)
637 messages.extend(checksum_lines)
638 logger.warning("\n".join(messages))
639
640 # We want to alert the user if a checksum is defined in the recipe but
641 # it does not match.
642 messages = []
643 messages.append("Checksum mismatch!")
644 bad_checksum = None
645
646 for ci in checksum_infos:
647 if ci["expected"] and ci["expected"] != ci["data"]:
648 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
649 "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
650 bad_checksum = ci["data"]
651
652 if bad_checksum:
653 messages.append("If this change is expected (e.g. you have upgraded " \
654 "to a new version without updating the checksums) " \
655 "then you can use these lines within the recipe:")
656 messages.extend(checksum_lines)
657 messages.append("Otherwise you should retry the download and/or " \
658 "check with upstream to determine if the file has " \
659 "become corrupted or otherwise unexpectedly modified.")
660 raise ChecksumError("\n".join(messages), ud.url, bad_checksum)
661
662 return checksum_dict
663
664def verify_donestamp(ud, d, origud=None):
665 """
666 Check whether the done stamp file has the right checksums (if the fetch
667 method supports them). If it doesn't, delete the done stamp and force
668 a re-download.
669
670 Returns True, if the donestamp exists and is valid, False otherwise. When
671 returning False, any existing done stamps are removed.
672 """
673 if not ud.needdonestamp or (origud and not origud.needdonestamp):
674 return True
675
676 if not os.path.exists(ud.localpath):
677 # local path does not exist
678 if os.path.exists(ud.donestamp):
679 # done stamp exists, but the downloaded file does not; the done stamp
680 # must be incorrect, re-trigger the download
681 bb.utils.remove(ud.donestamp)
682 return False
683
684 if (not ud.method.supports_checksum(ud) or
685 (origud and not origud.method.supports_checksum(origud))):
686 # if done stamp exists and checksums not supported; assume the local
687 # file is current
688 return os.path.exists(ud.donestamp)
689
690 precomputed_checksums = {}
691 # Only re-use the precomputed checksums if the donestamp is newer than the
692 # file. Do not rely on the mtime of directories, though. If ud.localpath is
693 # a directory, there will probably not be any checksums anyway.
694 if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or
695 os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)):
696 try:
697 with open(ud.donestamp, "rb") as cachefile:
698 pickled = pickle.Unpickler(cachefile)
699 precomputed_checksums.update(pickled.load())
700 except Exception as e:
701 # Avoid the warnings on the upgrade path from emtpy done stamp
702 # files to those containing the checksums.
703 if not isinstance(e, EOFError):
704 # Ignore errors, they aren't fatal
705 logger.warning("Couldn't load checksums from donestamp %s: %s "
706 "(msg: %s)" % (ud.donestamp, type(e).__name__,
707 str(e)))
708
709 try:
710 checksums = verify_checksum(ud, d, precomputed_checksums)
711 # If the cache file did not have the checksums, compute and store them
712 # as an upgrade path from the previous done stamp file format.
713 if checksums != precomputed_checksums:
714 with open(ud.donestamp, "wb") as cachefile:
715 p = pickle.Pickler(cachefile, 2)
716 p.dump(checksums)
717 return True
718 except ChecksumError as e:
719 # Checksums failed to verify, trigger re-download and remove the
720 # incorrect stamp file.
721 logger.warning("Checksum mismatch for local file %s\n"
722 "Cleaning and trying again." % ud.localpath)
723 if os.path.exists(ud.localpath):
724 rename_bad_checksum(ud, e.checksum)
725 bb.utils.remove(ud.donestamp)
726 return False
727
728
729def update_stamp(ud, d):
730 """
731 donestamp is file stamp indicating the whole fetching is done
732 this function update the stamp after verifying the checksum
733 """
734 if not ud.needdonestamp:
735 return
736
737 if os.path.exists(ud.donestamp):
738 # Touch the done stamp file to show active use of the download
739 try:
740 os.utime(ud.donestamp, None)
741 except:
742 # Errors aren't fatal here
743 pass
744 else:
745 try:
746 checksums = verify_checksum(ud, d)
747 # Store the checksums for later re-verification against the recipe
748 with open(ud.donestamp, "wb") as cachefile:
749 p = pickle.Pickler(cachefile, 2)
750 p.dump(checksums)
751 except ChecksumError as e:
752 # Checksums failed to verify, trigger re-download and remove the
753 # incorrect stamp file.
754 logger.warning("Checksum mismatch for local file %s\n"
755 "Cleaning and trying again." % ud.localpath)
756 if os.path.exists(ud.localpath):
757 rename_bad_checksum(ud, e.checksum)
758 bb.utils.remove(ud.donestamp)
759 raise
760
761def subprocess_setup():
762 # Python installs a SIGPIPE handler by default. This is usually not what
763 # non-Python subprocesses expect.
764 # SIGPIPE errors are known issues with gzip/bash
765 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
766
767def mark_recipe_nocache(d):
768 if d.getVar('BB_SRCREV_POLICY') != "cache":
769 d.setVar('BB_DONT_CACHE', '1')
770
771def get_autorev(d):
772 mark_recipe_nocache(d)
773 d.setVar("__BBAUTOREV_SEEN", True)
774 return "AUTOINC"
775
776def _get_srcrev(d, method_name='sortable_revision'):
777 """
778 Return the revision string, usually for use in the version string (PV) of the current package
779 Most packages usually only have one SCM so we just pass on the call.
780 In the multi SCM case, we build a value based on SRCREV_FORMAT which must
781 have been set.
782
783 The idea here is that we put the string "AUTOINC+" into return value if the revisions are not
784 incremental, other code is then responsible for turning that into an increasing value (if needed)
785
786 A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if
787 that fetcher provides a method with the given name and the same signature as sortable_revision.
788 """
789
790 d.setVar("__BBSRCREV_SEEN", "1")
791 recursion = d.getVar("__BBINSRCREV")
792 if recursion:
793 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
794 d.setVar("__BBINSRCREV", True)
795
796 scms = []
797 revs = []
798 fetcher = Fetch(d.getVar('SRC_URI').split(), d)
799 urldata = fetcher.ud
800 for u in urldata:
801 if urldata[u].method.supports_srcrev():
802 scms.append(u)
803
804 if not scms:
805 d.delVar("__BBINSRCREV")
806 return "", revs
807
808
809 if len(scms) == 1:
810 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
811 revs.append(rev)
812 if len(rev) > 10:
813 rev = rev[:10]
814 d.delVar("__BBINSRCREV")
815 if autoinc:
816 return "AUTOINC+" + rev, revs
817 return rev, revs
818
819 #
820 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
821 #
822 format = d.getVar('SRCREV_FORMAT')
823 if not format:
824 raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\
825 "The SCMs are:\n%s" % '\n'.join(scms))
826
827 name_to_rev = {}
828 seenautoinc = False
829 for scm in scms:
830 ud = urldata[scm]
831 autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
832 revs.append(rev)
833 seenautoinc = seenautoinc or autoinc
834 if len(rev) > 10:
835 rev = rev[:10]
836 name_to_rev[ud.name] = rev
837 # Replace names by revisions in the SRCREV_FORMAT string. The approach used
838 # here can handle names being prefixes of other names and names appearing
839 # as substrings in revisions (in which case the name should not be
840 # expanded). The '|' regular expression operator tries matches from left to
841 # right, so we need to sort the names with the longest ones first.
842 names_descending_len = sorted(name_to_rev, key=len, reverse=True)
843 name_to_rev_re = "|".join(re.escape(name) for name in names_descending_len)
844 format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format)
845
846 if seenautoinc:
847 format = "AUTOINC+" + format
848
849 d.delVar("__BBINSRCREV")
850 return format, revs
851
852def get_hashvalue(d, method_name='sortable_revision'):
853 pkgv, revs = _get_srcrev(d, method_name=method_name)
854 return " ".join(revs)
855
856def get_pkgv_string(d, method_name='sortable_revision'):
857 pkgv, revs = _get_srcrev(d, method_name=method_name)
858 return pkgv
859
860def get_srcrev(d, method_name='sortable_revision'):
861 pkgv, revs = _get_srcrev(d, method_name=method_name)
862 if not pkgv:
863 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
864 return pkgv
865
866def localpath(url, d):
867 fetcher = bb.fetch2.Fetch([url], d)
868 return fetcher.localpath(url)
869
870# Need to export PATH as binary could be in metadata paths
871# rather than host provided
872# Also include some other variables.
873FETCH_EXPORT_VARS = ['HOME', 'PATH',
874 'HTTP_PROXY', 'http_proxy',
875 'HTTPS_PROXY', 'https_proxy',
876 'FTP_PROXY', 'ftp_proxy',
877 'FTPS_PROXY', 'ftps_proxy',
878 'NO_PROXY', 'no_proxy',
879 'ALL_PROXY', 'all_proxy',
880 'GIT_PROXY_COMMAND',
881 'GIT_SSH',
882 'GIT_SSH_COMMAND',
883 'GIT_SSL_CAINFO',
884 'GIT_SMART_HTTP',
885 'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
886 'SOCKS5_USER', 'SOCKS5_PASSWD',
887 'DBUS_SESSION_BUS_ADDRESS',
888 'P4CONFIG',
889 'SSL_CERT_FILE',
890 'NODE_EXTRA_CA_CERTS',
891 'AWS_PROFILE',
892 'AWS_ACCESS_KEY_ID',
893 'AWS_SECRET_ACCESS_KEY',
894 'AWS_ROLE_ARN',
895 'AWS_WEB_IDENTITY_TOKEN_FILE',
896 'AWS_DEFAULT_REGION',
897 'AWS_SESSION_TOKEN',
898 'GIT_CACHE_PATH',
899 'REMOTE_CONTAINERS_IPC',
900 'GITHUB_TOKEN',
901 'SSL_CERT_DIR']
902
903def get_fetcher_environment(d):
904 newenv = {}
905 origenv = d.getVar("BB_ORIGENV")
906 for name in bb.fetch2.FETCH_EXPORT_VARS:
907 value = d.getVar(name)
908 if not value and origenv:
909 value = origenv.getVar(name)
910 if value:
911 newenv[name] = value
912 return newenv
913
914def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
915 """
916 Run cmd returning the command output
917 Raise an error if interrupted or cmd fails
918 Optionally echo command output to stdout
919 Optionally remove the files/directories listed in cleanup upon failure
920 """
921
922 exportvars = FETCH_EXPORT_VARS
923
924 if not cleanup:
925 cleanup = []
926
927 # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we
928 # can end up in circular recursion here so give the option of breaking it
929 # in a data store copy.
930 try:
931 d.getVar("PV")
932 d.getVar("PR")
933 except bb.data_smart.ExpansionError:
934 d = bb.data.createCopy(d)
935 d.setVar("PV", "fetcheravoidrecurse")
936 d.setVar("PR", "fetcheravoidrecurse")
937
938 origenv = d.getVar("BB_ORIGENV", False)
939 for var in exportvars:
940 val = d.getVar(var) or (origenv and origenv.getVar(var))
941 if val:
942 cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
943
944 # Disable pseudo as it may affect ssh, potentially causing it to hang.
945 cmd = 'export PSEUDO_DISABLED=1; ' + cmd
946
947 if workdir:
948 logger.debug("Running '%s' in %s" % (cmd, workdir))
949 else:
950 logger.debug("Running %s", cmd)
951
952 success = False
953 error_message = ""
954
955 try:
956 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
957 success = True
958 except bb.process.NotFoundError as e:
959 error_message = "Fetch command %s not found" % (e.command)
960 except bb.process.ExecutionError as e:
961 if e.stdout:
962 output = "output:\n%s\n%s" % (e.stdout, e.stderr)
963 elif e.stderr:
964 output = "output:\n%s" % e.stderr
965 else:
966 if log:
967 output = "see logfile for output"
968 else:
969 output = "no output"
970 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
971 except bb.process.CmdError as e:
972 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
973 if not success:
974 for f in cleanup:
975 try:
976 bb.utils.remove(f, True)
977 except OSError:
978 pass
979
980 raise FetchError(error_message)
981
982 return output
983
984def check_network_access(d, info, url):
985 """
986 log remote network access, and error if BB_NO_NETWORK is set or the given
987 URI is untrusted
988 """
989 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
990 raise NetworkAccess(url, info)
991 elif not trusted_network(d, url):
992 raise UntrustedUrl(url, info)
993 else:
994 logger.debug("Fetcher accessed the network with the command %s" % info)
995
996def build_mirroruris(origud, mirrors, ld):
997 uris = []
998 uds = []
999
1000 replacements = {}
1001 replacements["TYPE"] = origud.type
1002 replacements["HOST"] = origud.host
1003 replacements["PATH"] = origud.path
1004 replacements["BASENAME"] = origud.path.split("/")[-1]
1005 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
1006
1007 def adduri(ud, uris, uds, mirrors, tarballs):
1008 for line in mirrors:
1009 try:
1010 (find, replace) = line
1011 except ValueError:
1012 continue
1013
1014 for tarball in tarballs:
1015 newuri = uri_replace(ud, find, replace, replacements, ld, tarball)
1016 if not newuri or newuri in uris or newuri == origud.url:
1017 continue
1018
1019 if not trusted_network(ld, newuri):
1020 logger.debug("Mirror %s not in the list of trusted networks, skipping" % (newuri))
1021 continue
1022
1023 # Create a local copy of the mirrors minus the current line
1024 # this will prevent us from recursively processing the same line
1025 # as well as indirect recursion A -> B -> C -> A
1026 localmirrors = list(mirrors)
1027 localmirrors.remove(line)
1028
1029 try:
1030 newud = FetchData(newuri, ld)
1031 newud.ignore_checksums = True
1032 newud.setup_localpath(ld)
1033 except bb.fetch2.BBFetchException as e:
1034 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
1035 logger.debug(str(e))
1036 try:
1037 # setup_localpath of file:// urls may fail, we should still see
1038 # if mirrors of the url exist
1039 adduri(newud, uris, uds, localmirrors, tarballs)
1040 except UnboundLocalError:
1041 pass
1042 continue
1043 uris.append(newuri)
1044 uds.append(newud)
1045
1046 adduri(newud, uris, uds, localmirrors, tarballs)
1047
1048 adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None])
1049
1050 return uris, uds
1051
1052def rename_bad_checksum(ud, suffix):
1053 """
1054 Renames files to have suffix from parameter
1055 """
1056
1057 if ud.localpath is None:
1058 return
1059
1060 new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix)
1061 bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath))
1062 if not bb.utils.movefile(ud.localpath, new_localpath):
1063 bb.warn("Renaming %s to %s failed, grep movefile in log.do_fetch to see why" % (ud.localpath, new_localpath))
1064
1065
1066def try_mirror_url(fetch, origud, ud, ld, check = False):
1067 # Return of None or a value means we're finished
1068 # False means try another url
1069
1070 if ud.lockfile and ud.lockfile != origud.lockfile:
1071 lf = bb.utils.lockfile(ud.lockfile)
1072
1073 try:
1074 if check:
1075 found = ud.method.checkstatus(fetch, ud, ld)
1076 if found:
1077 return found
1078 return False
1079
1080 if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld):
1081 ud.method.download(ud, ld)
1082 if hasattr(ud.method,"build_mirror_data"):
1083 ud.method.build_mirror_data(ud, ld)
1084
1085 if not ud.localpath or not os.path.exists(ud.localpath):
1086 return False
1087
1088 if ud.localpath == origud.localpath:
1089 return ud.localpath
1090
1091 # We may be obtaining a mirror tarball which needs further processing by the real fetcher
1092 # If that tarball is a local file:// we need to provide a symlink to it
1093 dldir = ld.getVar("DL_DIR")
1094
1095 if bb.utils.to_boolean(ld.getVar("BB_FETCH_PREMIRRORONLY")):
1096 ld = ld.createCopy()
1097 ld.setVar("BB_NO_NETWORK", "1")
1098
1099 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
1100 # Create donestamp in old format to avoid triggering a re-download
1101 if ud.donestamp:
1102 bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
1103 open(ud.donestamp, 'w').close()
1104 dest = os.path.join(dldir, os.path.basename(ud.localpath))
1105 if not os.path.exists(dest):
1106 # In case this is executing without any file locks held (as is
1107 # the case for file:// URLs), two tasks may end up here at the
1108 # same time, in which case we do not want the second task to
1109 # fail when the link has already been created by the first task.
1110 try:
1111 os.symlink(ud.localpath, dest)
1112 except FileExistsError:
1113 pass
1114 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
1115 origud.method.download(origud, ld)
1116 if hasattr(origud.method, "build_mirror_data"):
1117 origud.method.build_mirror_data(origud, ld)
1118 return origud.localpath
1119 # Otherwise the result is a local file:// and we symlink to it
1120 # This may also be a link to a shallow archive
1121 # When using shallow mode, add a symlink to the original fullshallow
1122 # path to ensure a valid symlink even in the `PREMIRRORS` case
1123 origud.method.update_mirror_links(ud, origud)
1124 update_stamp(origud, ld)
1125 return ud.localpath
1126
1127 except bb.fetch2.NetworkAccess:
1128 raise
1129
1130 except IOError as e:
1131 if e.errno in [errno.ESTALE]:
1132 logger.warning("Stale Error Observed %s." % ud.url)
1133 return False
1134 raise
1135
1136 except bb.fetch2.BBFetchException as e:
1137 if isinstance(e, ChecksumError):
1138 logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url))
1139 logger.warning(str(e))
1140 if os.path.exists(ud.localpath):
1141 rename_bad_checksum(ud, e.checksum)
1142 elif isinstance(e, NoChecksumError):
1143 raise
1144 else:
1145 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
1146 logger.debug(str(e))
1147 try:
1148 if ud.method.cleanup_upon_failure():
1149 ud.method.clean(ud, ld)
1150 except UnboundLocalError:
1151 pass
1152 return False
1153 finally:
1154 if ud.lockfile and ud.lockfile != origud.lockfile:
1155 bb.utils.unlockfile(lf)
1156
1157def try_mirrors(fetch, d, origud, mirrors, check = False):
1158 """
1159 Try to use a mirrored version of the sources.
1160 This method will be automatically called before the fetchers go.
1161
1162 d Is a bb.data instance
1163 uri is the original uri we're trying to download
1164 mirrors is the list of mirrors we're going to try
1165 """
1166 ld = d.createCopy()
1167
1168 uris, uds = build_mirroruris(origud, mirrors, ld)
1169
1170 for index, uri in enumerate(uris):
1171 ret = try_mirror_url(fetch, origud, uds[index], ld, check)
1172 if ret:
1173 return ret
1174 return None
1175
1176def trusted_network(d, url):
1177 """
1178 Use a trusted url during download if networking is enabled and
1179 BB_ALLOWED_NETWORKS is set globally or for a specific recipe.
1180 Note: modifies SRC_URI & mirrors.
1181 """
1182 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
1183 return True
1184
1185 pkgname = d.getVar('PN')
1186 trusted_hosts = None
1187 if pkgname:
1188 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
1189
1190 if not trusted_hosts:
1191 trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS')
1192
1193 # Not enabled.
1194 if not trusted_hosts:
1195 return True
1196
1197 scheme, network, path, user, passwd, param = decodeurl(url)
1198
1199 if not network:
1200 return True
1201
1202 network = network.split(':')[0]
1203 network = network.lower()
1204
1205 for host in trusted_hosts.split(" "):
1206 host = host.lower()
1207 if host.startswith("*.") and ("." + network).endswith(host[1:]):
1208 return True
1209 if host == network:
1210 return True
1211
1212 return False
1213
1214def srcrev_internal_helper(ud, d, name):
1215 """
1216 Return:
1217 a) a source revision if specified
1218 b) latest revision if SRCREV="AUTOINC"
1219 c) None if not specified
1220 """
1221
1222 srcrev = None
1223 pn = d.getVar("PN")
1224 attempts = []
1225 if name != '' and pn:
1226 attempts.append("SRCREV_%s:pn-%s" % (name, pn))
1227 if name != '':
1228 attempts.append("SRCREV_%s" % name)
1229 if pn:
1230 attempts.append("SRCREV:pn-%s" % pn)
1231 attempts.append("SRCREV")
1232
1233 for a in attempts:
1234 srcrev = d.getVar(a)
1235 if srcrev and srcrev != "INVALID":
1236 break
1237
1238 if 'rev' in ud.parm:
1239 parmrev = ud.parm['rev']
1240 if srcrev == "INVALID" or not srcrev:
1241 return parmrev
1242 if srcrev != parmrev:
1243 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev))
1244 return parmrev
1245
1246 if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev):
1247 return ud.parm['tag']
1248
1249 if srcrev == "INVALID" or not srcrev:
1250 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
1251 if srcrev == "AUTOINC":
1252 d.setVar("__BBAUTOREV_ACTED_UPON", True)
1253 srcrev = ud.method.latest_revision(ud, d, name)
1254
1255 return srcrev
1256
1257def get_checksum_file_list(d):
1258 """ Get a list of files checksum in SRC_URI
1259
1260 Returns the resolved local paths of all local file entries in
1261 SRC_URI as a space-separated string
1262 """
1263 fetch = Fetch([], d, cache = False, localonly = True)
1264 filelist = []
1265 for u in fetch.urls:
1266 ud = fetch.ud[u]
1267 if ud and isinstance(ud.method, local.Local):
1268 found = False
1269 paths = ud.method.localfile_searchpaths(ud, d)
1270 for f in paths:
1271 pth = ud.path
1272 if os.path.exists(f):
1273 found = True
1274 filelist.append(f + ":" + str(os.path.exists(f)))
1275 if not found:
1276 bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
1277 "\nThe following paths were searched:"
1278 "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
1279
1280 return " ".join(filelist)
1281
1282def get_file_checksums(filelist, pn, localdirsexclude):
1283 """Get a list of the checksums for a list of local files
1284
1285 Returns the checksums for a list of local files, caching the results as
1286 it proceeds
1287
1288 """
1289 return _checksum_cache.get_checksums(filelist, pn, localdirsexclude)
1290
1291
1292class FetchData(object):
1293 """
1294 A class which represents the fetcher state for a given URI.
1295 """
1296 def __init__(self, url, d, localonly = False):
1297 # localpath is the location of a downloaded result. If not set, the file is local.
1298 self.donestamp = None
1299 self.needdonestamp = True
1300 self.localfile = ""
1301 self.localpath = None
1302 self.lockfile = None
1303 self.mirrortarballs = []
1304 self.basename = None
1305 self.basepath = None
1306 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
1307 self.date = self.getSRCDate(d)
1308 self.url = url
1309 if not self.user and "user" in self.parm:
1310 self.user = self.parm["user"]
1311 if not self.pswd and "pswd" in self.parm:
1312 self.pswd = self.parm["pswd"]
1313 self.setup = False
1314
1315 def configure_checksum(checksum_id):
1316 checksum_plain_name = "%ssum" % checksum_id
1317 if "name" in self.parm:
1318 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
1319 else:
1320 checksum_name = checksum_plain_name
1321
1322 if checksum_name in self.parm:
1323 checksum_expected = self.parm[checksum_name]
1324 elif checksum_plain_name in self.parm:
1325 checksum_expected = self.parm[checksum_plain_name]
1326 checksum_name = checksum_plain_name
1327 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]:
1328 checksum_expected = None
1329 else:
1330 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
1331
1332 setattr(self, "%s_name" % checksum_id, checksum_name)
1333 setattr(self, "%s_expected" % checksum_id, checksum_expected)
1334
1335 self.name = self.parm.get("name",'default')
1336 if "," in self.name:
1337 raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
1338
1339 self.method = None
1340 for m in methods:
1341 if m.supports(self, d):
1342 self.method = m
1343 break
1344
1345 if not self.method:
1346 raise NoMethodError(url)
1347
1348 if localonly and not isinstance(self.method, local.Local):
1349 raise NonLocalMethod()
1350
1351 if self.parm.get("proto", None) and "protocol" not in self.parm:
1352 logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN'))
1353 self.parm["protocol"] = self.parm.get("proto", None)
1354
1355 if hasattr(self.method, "urldata_init"):
1356 self.method.urldata_init(self, d)
1357
1358 for checksum_id in CHECKSUM_LIST:
1359 configure_checksum(checksum_id)
1360
1361 self.ignore_checksums = False
1362
1363 if "localpath" in self.parm:
1364 # if user sets localpath for file, use it instead.
1365 self.localpath = self.parm["localpath"]
1366 self.basename = os.path.basename(self.localpath)
1367 elif self.localfile:
1368 self.localpath = self.method.localpath(self, d)
1369
1370 dldir = d.getVar("DL_DIR")
1371
1372 if not self.needdonestamp:
1373 return
1374
1375 # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be.
1376 if self.localpath and self.localpath.startswith(dldir):
1377 basepath = self.localpath
1378 elif self.localpath:
1379 basepath = dldir + os.sep + os.path.basename(self.localpath)
1380 elif self.basepath or self.basename:
1381 basepath = dldir + os.sep + (self.basepath or self.basename)
1382 else:
1383 bb.fatal("Can't determine lock path for url %s" % url)
1384
1385 self.donestamp = basepath + '.done'
1386 self.lockfile = basepath + '.lock'
1387
1388 def setup_revisions(self, d):
1389 self.revision = srcrev_internal_helper(self, d, self.name)
1390
1391 def setup_localpath(self, d):
1392 if not self.localpath:
1393 self.localpath = self.method.localpath(self, d)
1394
1395 def getSRCDate(self, d):
1396 """
1397 Return the SRC Date for the component
1398
1399 d the bb.data module
1400 """
1401 if "srcdate" in self.parm:
1402 return self.parm['srcdate']
1403
1404 pn = d.getVar("PN")
1405
1406 if pn:
1407 return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE")
1408
1409 return d.getVar("SRCDATE") or d.getVar("DATE")
1410
1411class FetchMethod(object):
1412 """Base class for 'fetch'ing data"""
1413
1414 def __init__(self, urls=None):
1415 self.urls = []
1416
1417 def supports(self, urldata, d):
1418 """
1419 Check to see if this fetch class supports a given url.
1420 """
1421 return 0
1422
1423 def localpath(self, urldata, d):
1424 """
1425 Return the local filename of a given url assuming a successful fetch.
1426 Can also setup variables in urldata for use in go (saving code duplication
1427 and duplicate code execution)
1428 """
1429 return os.path.join(d.getVar("DL_DIR"), urldata.localfile)
1430
1431 def supports_checksum(self, urldata):
1432 """
1433 Is localpath something that can be represented by a checksum?
1434 """
1435
1436 # We cannot compute checksums for None
1437 if urldata.localpath is None:
1438 return False
1439 # We cannot compute checksums for directories
1440 if os.path.isdir(urldata.localpath):
1441 return False
1442 return True
1443
1444 def recommends_checksum(self, urldata):
1445 """
1446 Is the backend on where checksumming is recommended (should warnings
1447 be displayed if there is no checksum)?
1448 """
1449 return False
1450
1451 def cleanup_upon_failure(self):
1452 """
1453 When a fetch fails, should clean() be called?
1454 """
1455 return True
1456
1457 def verify_donestamp(self, ud, d):
1458 """
1459 Verify the donestamp file
1460 """
1461 return verify_donestamp(ud, d)
1462
1463 def update_donestamp(self, ud, d):
1464 """
1465 Update the donestamp file
1466 """
1467 update_stamp(ud, d)
1468
1469 def _strip_leading_slashes(self, relpath):
1470 """
1471 Remove leading slash as os.path.join can't cope
1472 """
1473 while os.path.isabs(relpath):
1474 relpath = relpath[1:]
1475 return relpath
1476
1477 def setUrls(self, urls):
1478 self.__urls = urls
1479
1480 def getUrls(self):
1481 return self.__urls
1482
1483 urls = property(getUrls, setUrls, None, "Urls property")
1484
1485 def need_update(self, ud, d):
1486 """
1487 Force a fetch, even if localpath exists?
1488 """
1489 if os.path.exists(ud.localpath):
1490 return False
1491 return True
1492
1493 def supports_srcrev(self):
1494 """
1495 The fetcher supports auto source revisions (SRCREV)
1496 """
1497 return False
1498
1499 def download(self, urldata, d):
1500 """
1501 Fetch urls
1502 Assumes localpath was called first
1503 """
1504 raise NoMethodError(urldata.url)
1505
1506 def unpack(self, urldata, rootdir, data):
1507 iterate = False
1508 file = urldata.localpath
1509
1510 try:
1511 unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True)
1512 except ValueError as exc:
1513 bb.fatal("Invalid value for 'unpack' parameter for %s: %s" %
1514 (file, urldata.parm.get('unpack')))
1515
1516 base, ext = os.path.splitext(file)
1517 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']:
1518 efile = os.path.join(rootdir, os.path.basename(base))
1519 else:
1520 efile = file
1521 cmd = None
1522
1523 if unpack:
1524 tar_cmd = 'tar --extract --no-same-owner'
1525 if 'striplevel' in urldata.parm:
1526 tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel']
1527 if file.endswith('.tar'):
1528 cmd = '%s -f %s' % (tar_cmd, file)
1529 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
1530 cmd = '%s -z -f %s' % (tar_cmd, file)
1531 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
1532 cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd)
1533 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
1534 cmd = 'gzip -dc %s > %s' % (file, efile)
1535 elif file.endswith('.bz2'):
1536 cmd = 'bzip2 -dc %s > %s' % (file, efile)
1537 elif file.endswith('.txz') or file.endswith('.tar.xz'):
1538 cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd)
1539 elif file.endswith('.xz'):
1540 cmd = 'xz -dc %s > %s' % (file, efile)
1541 elif file.endswith('.tar.lz'):
1542 cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd)
1543 elif file.endswith('.lz'):
1544 cmd = 'lzip -dc %s > %s' % (file, efile)
1545 elif file.endswith('.tar.7z'):
1546 cmd = '7z x -so %s | %s -f -' % (file, tar_cmd)
1547 elif file.endswith('.7z'):
1548 cmd = '7za x -y %s 1>/dev/null' % file
1549 elif file.endswith('.tzst') or file.endswith('.tar.zst'):
1550 cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd)
1551 elif file.endswith('.zst'):
1552 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
1553 elif file.endswith('.zip') or file.endswith('.jar'):
1554 try:
1555 dos = bb.utils.to_boolean(urldata.parm.get('dos'), False)
1556 except ValueError as exc:
1557 bb.fatal("Invalid value for 'dos' parameter for %s: %s" %
1558 (file, urldata.parm.get('dos')))
1559 cmd = 'unzip -q -o'
1560 if dos:
1561 cmd = '%s -a' % cmd
1562 cmd = "%s '%s'" % (cmd, file)
1563 elif file.endswith('.rpm') or file.endswith('.srpm'):
1564 if 'extract' in urldata.parm:
1565 unpack_file = urldata.parm.get('extract')
1566 cmd = 'rpm2cpio.sh %s | cpio -id %s' % (file, unpack_file)
1567 iterate = True
1568 iterate_file = unpack_file
1569 else:
1570 cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
1571 elif file.endswith('.deb') or file.endswith('.ipk'):
1572 output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup)
1573 datafile = None
1574 if output:
1575 for line in output.decode().splitlines():
1576 if line.startswith('data.tar.') or line == 'data.tar':
1577 datafile = line
1578 break
1579 else:
1580 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar* file", urldata.url)
1581 else:
1582 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
1583 cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
1584
1585 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
1586 if 'subdir' in urldata.parm:
1587 subdir = urldata.parm.get('subdir')
1588 if os.path.isabs(subdir):
1589 if not os.path.realpath(subdir).startswith(os.path.realpath(rootdir)):
1590 raise UnpackError("subdir argument isn't a subdirectory of unpack root %s" % rootdir, urldata.url)
1591 unpackdir = subdir
1592 else:
1593 unpackdir = os.path.join(rootdir, subdir)
1594 bb.utils.mkdirhier(unpackdir)
1595 else:
1596 unpackdir = rootdir
1597
1598 if not unpack or not cmd:
1599 urldata.unpack_tracer.unpack("file-copy", unpackdir)
1600 # If file == dest, then avoid any copies, as we already put the file into dest!
1601 dest = os.path.join(unpackdir, os.path.basename(file))
1602 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
1603 destdir = '.'
1604 # For file:// entries all intermediate dirs in path must be created at destination
1605 if urldata.type == "file":
1606 # Trailing '/' does a copying to wrong place
1607 urlpath = urldata.path.rstrip('/')
1608 # Want files places relative to cwd so no leading '/'
1609 urlpath = urlpath.lstrip('/')
1610 if urlpath.find("/") != -1:
1611 destdir = urlpath.rsplit("/", 1)[0] + '/'
1612 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
1613 cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir)
1614 else:
1615 urldata.unpack_tracer.unpack("archive-extract", unpackdir)
1616
1617 if not cmd:
1618 return
1619
1620 path = data.getVar('PATH')
1621 if path:
1622 cmd = "PATH=\"%s\" %s" % (path, cmd)
1623 bb.note("Unpacking %s to %s/" % (file, unpackdir))
1624 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=unpackdir)
1625
1626 if ret != 0:
1627 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url)
1628
1629 if iterate is True:
1630 iterate_urldata = urldata
1631 iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file)
1632 self.unpack(urldata, rootdir, data)
1633
1634 return
1635
1636 def clean(self, urldata, d):
1637 """
1638 Clean any existing full or partial download
1639 """
1640 bb.utils.remove(urldata.localpath)
1641
1642 def ensure_symlink(self, target, link_name):
1643 if not os.path.exists(link_name):
1644 dirname = os.path.dirname(link_name)
1645 bb.utils.mkdirhier(dirname)
1646 if os.path.islink(link_name):
1647 # Broken symbolic link
1648 os.unlink(link_name)
1649
1650 # In case this is executing without any file locks held (as is
1651 # the case for file:// URLs), two tasks may end up here at the
1652 # same time, in which case we do not want the second task to
1653 # fail when the link has already been created by the first task.
1654 try:
1655 os.symlink(target, link_name)
1656 except FileExistsError:
1657 pass
1658
1659 def update_mirror_links(self, ud, origud):
1660 # For local file:// results, create a symlink to them
1661 # This may also be a link to a shallow archive
1662 self.ensure_symlink(ud.localpath, origud.localpath)
1663
1664 def try_premirror(self, urldata, d):
1665 """
1666 Should premirrors be used?
1667 """
1668 return True
1669
1670 def try_mirrors(self, fetch, urldata, d, mirrors, check=False):
1671 """
1672 Try to use a mirror
1673 """
1674 return bool(try_mirrors(fetch, d, urldata, mirrors, check))
1675
1676 def checkstatus(self, fetch, urldata, d):
1677 """
1678 Check the status of a URL
1679 Assumes localpath was called first
1680 """
1681 logger.info("URL %s could not be checked for status since no method exists.", urldata.url)
1682 return True
1683
1684 def latest_revision(self, ud, d, name):
1685 """
1686 Look in the cache for the latest revision, if not present ask the SCM.
1687 """
1688 if not hasattr(self, "_latest_revision"):
1689 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
1690
1691 key = self.generate_revision_key(ud, d, name)
1692
1693 rev = _revisions_cache.get_rev(key)
1694 if rev is None:
1695 rev = self._latest_revision(ud, d, name)
1696 _revisions_cache.set_rev(key, rev)
1697 return rev
1698
1699 def sortable_revision(self, ud, d, name):
1700 latest_rev = self._build_revision(ud, d, name)
1701 return True, str(latest_rev)
1702
1703 def generate_revision_key(self, ud, d, name):
1704 return self._revision_key(ud, d, name)
1705
1706 def latest_versionstring(self, ud, d):
1707 """
1708 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
1709 by searching through the tags output of ls-remote, comparing
1710 versions and returning the highest match as a (version, revision) pair.
1711 """
1712 return ('', '')
1713
1714 def done(self, ud, d):
1715 """
1716 Is the download done ?
1717 """
1718 if os.path.exists(ud.localpath):
1719 return True
1720 return False
1721
1722 def implicit_urldata(self, ud, d):
1723 """
1724 Get a list of FetchData objects for any implicit URLs that will also
1725 be downloaded when we fetch the given URL.
1726 """
1727 return []
1728
1729
1730class DummyUnpackTracer(object):
1731 """
1732 Abstract API definition for a class that traces unpacked source files back
1733 to their respective upstream SRC_URI entries, for software composition
1734 analysis, license compliance and detailed SBOM generation purposes.
1735 User may load their own unpack tracer class (instead of the dummy
1736 one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
1737 """
1738 def start(self, unpackdir, urldata_dict, d):
1739 """
1740 Start tracing the core Fetch.unpack process, using an index to map
1741 unpacked files to each SRC_URI entry.
1742 This method is called by Fetch.unpack and it may receive nested calls by
1743 gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
1744 URLs and by recursively calling Fetch.unpack from new (nested) Fetch
1745 instances.
1746 """
1747 return
1748 def start_url(self, url):
1749 """Start tracing url unpack process.
1750 This method is called by Fetch.unpack before the fetcher-specific unpack
1751 method starts, and it may receive nested calls by gitsm and npmsw
1752 fetchers.
1753 """
1754 return
1755 def unpack(self, unpack_type, destdir):
1756 """
1757 Set unpack_type and destdir for current url.
1758 This method is called by the fetcher-specific unpack method after url
1759 tracing started.
1760 """
1761 return
1762 def finish_url(self, url):
1763 """Finish tracing url unpack process and update the file index.
1764 This method is called by Fetch.unpack after the fetcher-specific unpack
1765 method finished its job, and it may receive nested calls by gitsm
1766 and npmsw fetchers.
1767 """
1768 return
1769 def complete(self):
1770 """
1771 Finish tracing the Fetch.unpack process, and check if all nested
1772 Fecth.unpack calls (if any) have been completed; if so, save collected
1773 metadata.
1774 """
1775 return
1776
1777
1778class Fetch(object):
1779 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
1780 if localonly and cache:
1781 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
1782
1783 if not urls:
1784 urls = d.getVar("SRC_URI").split()
1785 self.urls = urls
1786 self.d = d
1787 self.ud = {}
1788 self.connection_cache = connection_cache
1789
1790 fn = d.getVar('FILE')
1791 mc = d.getVar('__BBMULTICONFIG') or ""
1792 key = None
1793 if cache and fn:
1794 key = mc + fn + str(id(d))
1795 if key in urldata_cache:
1796 self.ud = urldata_cache[key]
1797
1798 # the unpack_tracer object needs to be made available to possible nested
1799 # Fetch instances (when those are created by gitsm and npmsw fetchers)
1800 # so we set it as a global variable
1801 global unpack_tracer
1802 try:
1803 unpack_tracer
1804 except NameError:
1805 class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
1806 if class_path:
1807 # use user-defined unpack tracer class
1808 import importlib
1809 module_name, _, class_name = class_path.rpartition(".")
1810 module = importlib.import_module(module_name)
1811 class_ = getattr(module, class_name)
1812 unpack_tracer = class_()
1813 else:
1814 # fall back to the dummy/abstract class
1815 unpack_tracer = DummyUnpackTracer()
1816
1817 for url in urls:
1818 if url not in self.ud:
1819 try:
1820 self.ud[url] = FetchData(url, d, localonly)
1821 self.ud[url].unpack_tracer = unpack_tracer
1822 except NonLocalMethod:
1823 if localonly:
1824 self.ud[url] = None
1825 pass
1826
1827 if key:
1828 urldata_cache[key] = self.ud
1829
1830 def localpath(self, url):
1831 if url not in self.urls:
1832 self.ud[url] = FetchData(url, self.d)
1833
1834 self.ud[url].setup_localpath(self.d)
1835 return self.ud[url].localpath
1836
1837 def localpaths(self):
1838 """
1839 Return a list of the local filenames, assuming successful fetch
1840 """
1841 local = []
1842
1843 for u in self.urls:
1844 ud = self.ud[u]
1845 ud.setup_localpath(self.d)
1846 local.append(ud.localpath)
1847
1848 return local
1849
1850 def download(self, urls=None):
1851 """
1852 Fetch all urls
1853 """
1854 if not urls:
1855 urls = self.urls
1856
1857 network = self.d.getVar("BB_NO_NETWORK")
1858 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
1859
1860 checksum_missing_messages = []
1861 for u in urls:
1862 ud = self.ud[u]
1863 ud.setup_localpath(self.d)
1864 m = ud.method
1865 done = False
1866
1867 if ud.lockfile:
1868 lf = bb.utils.lockfile(ud.lockfile)
1869
1870 try:
1871 self.d.setVar("BB_NO_NETWORK", network)
1872 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
1873 done = True
1874 elif m.try_premirror(ud, self.d):
1875 logger.debug("Trying PREMIRRORS")
1876 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
1877 done = m.try_mirrors(self, ud, self.d, mirrors)
1878 if done:
1879 try:
1880 # early checksum verification so that if the checksum of the premirror
1881 # contents mismatch the fetcher can still try upstream and mirrors
1882 m.update_donestamp(ud, self.d)
1883 except ChecksumError as e:
1884 logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
1885 logger.debug(str(e))
1886 done = False
1887
1888 d = self.d
1889 if premirroronly:
1890 # Only disable the network in a copy
1891 d = bb.data.createCopy(self.d)
1892 d.setVar("BB_NO_NETWORK", "1")
1893
1894 firsterr = None
1895 verified_stamp = False
1896 if done:
1897 verified_stamp = m.verify_donestamp(ud, d)
1898 if not done and (not verified_stamp or m.need_update(ud, d)):
1899 try:
1900 if not trusted_network(d, ud.url):
1901 raise UntrustedUrl(ud.url)
1902 logger.debug("Trying Upstream")
1903 m.download(ud, d)
1904 if hasattr(m, "build_mirror_data"):
1905 m.build_mirror_data(ud, d)
1906 done = True
1907 # early checksum verify, so that if checksum mismatched,
1908 # fetcher still have chance to fetch from mirror
1909 m.update_donestamp(ud, d)
1910
1911 except bb.fetch2.NetworkAccess:
1912 raise
1913
1914 except BBFetchException as e:
1915 if isinstance(e, ChecksumError):
1916 logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u)
1917 logger.debug(str(e))
1918 if os.path.exists(ud.localpath):
1919 rename_bad_checksum(ud, e.checksum)
1920 elif isinstance(e, NoChecksumError):
1921 raise
1922 else:
1923 logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u)
1924 logger.debug(str(e))
1925 firsterr = e
1926 # Remove any incomplete fetch
1927 if not verified_stamp and m.cleanup_upon_failure():
1928 m.clean(ud, d)
1929 logger.debug("Trying MIRRORS")
1930 mirrors = mirror_from_string(d.getVar('MIRRORS'))
1931 done = m.try_mirrors(self, ud, d, mirrors)
1932
1933 if not done or not m.done(ud, d):
1934 if firsterr:
1935 logger.error(str(firsterr))
1936 raise FetchError("Unable to fetch URL from any source.", u)
1937
1938 m.update_donestamp(ud, d)
1939
1940 except IOError as e:
1941 if e.errno in [errno.ESTALE]:
1942 logger.error("Stale Error Observed %s." % u)
1943 raise ChecksumError("Stale Error Detected")
1944
1945 except BBFetchException as e:
1946 if isinstance(e, NoChecksumError):
1947 (message, _) = e.args
1948 checksum_missing_messages.append(message)
1949 continue
1950 elif isinstance(e, ChecksumError):
1951 logger.error("Checksum failure fetching %s" % u)
1952 raise
1953
1954 finally:
1955 if ud.lockfile:
1956 bb.utils.unlockfile(lf)
1957 if checksum_missing_messages:
1958 logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
1959 raise BBFetchException("There was some missing checksums in the recipe")
1960
1961 def checkstatus(self, urls=None):
1962 """
1963 Check all URLs exist upstream.
1964
1965 Returns None if the URLs exist, raises FetchError if the check wasn't
1966 successful but there wasn't an error (such as file not found), and
1967 raises other exceptions in error cases.
1968 """
1969
1970 if not urls:
1971 urls = self.urls
1972
1973 for u in urls:
1974 ud = self.ud[u]
1975 ud.setup_localpath(self.d)
1976 m = ud.method
1977 logger.debug("Testing URL %s", u)
1978 # First try checking uri, u, from PREMIRRORS
1979 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
1980 ret = m.try_mirrors(self, ud, self.d, mirrors, True)
1981 if not ret:
1982 # Next try checking from the original uri, u
1983 ret = m.checkstatus(self, ud, self.d)
1984 if not ret:
1985 # Finally, try checking uri, u, from MIRRORS
1986 mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
1987 ret = m.try_mirrors(self, ud, self.d, mirrors, True)
1988
1989 if not ret:
1990 raise FetchError("URL doesn't work", u)
1991
1992 def unpack(self, root, urls=None):
1993 """
1994 Unpack urls to root
1995 """
1996
1997 if not urls:
1998 urls = self.urls
1999
2000 unpack_tracer.start(root, self.ud, self.d)
2001
2002 for u in urls:
2003 ud = self.ud[u]
2004 ud.setup_localpath(self.d)
2005
2006 if ud.lockfile:
2007 lf = bb.utils.lockfile(ud.lockfile)
2008
2009 unpack_tracer.start_url(u)
2010 ud.method.unpack(ud, root, self.d)
2011 unpack_tracer.finish_url(u)
2012
2013 if ud.lockfile:
2014 bb.utils.unlockfile(lf)
2015
2016 unpack_tracer.complete()
2017
2018 def clean(self, urls=None):
2019 """
2020 Clean files that the fetcher gets or places
2021 """
2022
2023 if not urls:
2024 urls = self.urls
2025
2026 for url in urls:
2027 if url not in self.ud:
2028 self.ud[url] = FetchData(url, self.d)
2029 ud = self.ud[url]
2030 ud.setup_localpath(self.d)
2031
2032 if not ud.localfile and ud.localpath is None:
2033 continue
2034
2035 if ud.lockfile:
2036 lf = bb.utils.lockfile(ud.lockfile)
2037
2038 ud.method.clean(ud, self.d)
2039 if ud.donestamp:
2040 bb.utils.remove(ud.donestamp)
2041
2042 if ud.lockfile:
2043 bb.utils.unlockfile(lf)
2044
2045 def expanded_urldata(self, urls=None):
2046 """
2047 Get an expanded list of FetchData objects covering both the given
2048 URLS and any additional implicit URLs that are added automatically by
2049 the appropriate FetchMethod.
2050 """
2051
2052 if not urls:
2053 urls = self.urls
2054
2055 urldata = []
2056 for url in urls:
2057 ud = self.ud[url]
2058 urldata.append(ud)
2059 urldata += ud.method.implicit_urldata(ud, self.d)
2060
2061 return urldata
2062
2063class FetchConnectionCache(object):
2064 """
2065 A class which represents an container for socket connections.
2066 """
2067 def __init__(self):
2068 self.cache = {}
2069
2070 def get_connection_name(self, host, port):
2071 return host + ':' + str(port)
2072
2073 def add_connection(self, host, port, connection):
2074 cn = self.get_connection_name(host, port)
2075
2076 if cn not in self.cache:
2077 self.cache[cn] = connection
2078
2079 def get_connection(self, host, port):
2080 connection = None
2081
2082 cn = self.get_connection_name(host, port)
2083 if cn in self.cache:
2084 connection = self.cache[cn]
2085
2086 return connection
2087
2088 def remove_connection(self, host, port):
2089 cn = self.get_connection_name(host, port)
2090 if cn in self.cache:
2091 self.cache[cn].close()
2092 del self.cache[cn]
2093
2094 def close_connections(self):
2095 for cn in list(self.cache.keys()):
2096 self.cache[cn].close()
2097 del self.cache[cn]
2098
2099from . import cvs
2100from . import git
2101from . import gitsm
2102from . import gitannex
2103from . import local
2104from . import svn
2105from . import wget
2106from . import ssh
2107from . import sftp
2108from . import s3
2109from . import perforce
2110from . import bzr
2111from . import hg
2112from . import osc
2113from . import repo
2114from . import clearcase
2115from . import npm
2116from . import npmsw
2117from . import az
2118from . import crate
2119from . import gcp
2120from . import gomod
2121
2122methods.append(local.Local())
2123methods.append(wget.Wget())
2124methods.append(svn.Svn())
2125methods.append(git.Git())
2126methods.append(gitsm.GitSM())
2127methods.append(gitannex.GitANNEX())
2128methods.append(cvs.Cvs())
2129methods.append(ssh.SSH())
2130methods.append(sftp.SFTP())
2131methods.append(s3.S3())
2132methods.append(perforce.Perforce())
2133methods.append(bzr.Bzr())
2134methods.append(hg.Hg())
2135methods.append(osc.Osc())
2136methods.append(repo.Repo())
2137methods.append(clearcase.ClearCase())
2138methods.append(npm.Npm())
2139methods.append(npmsw.NpmShrinkWrap())
2140methods.append(az.Az())
2141methods.append(crate.Crate())
2142methods.append(gcp.GCP())
2143methods.append(gomod.GoMod())
2144methods.append(gomod.GoModGit())
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py
deleted file mode 100644
index 1d3664f213..0000000000
--- a/bitbake/lib/bb/fetch2/az.py
+++ /dev/null
@@ -1,98 +0,0 @@
1"""
2BitBake 'Fetch' Azure Storage implementation
3
4"""
5
6# Copyright (C) 2021 Alejandro Hernandez Samaniego
7#
8# Based on bb.fetch2.wget:
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import shlex
16import os
17import bb
18from bb.fetch2 import FetchError
19from bb.fetch2 import logger
20from bb.fetch2.wget import Wget
21
22
23class Az(Wget):
24
25 def supports(self, ud, d):
26 """
27 Check to see if a given url can be fetched from Azure Storage
28 """
29 return ud.type in ['az']
30
31
32 def checkstatus(self, fetch, ud, d, try_again=True):
33
34 # checkstatus discards parameters either way, we need to do this before adding the SAS
35 ud.url = ud.url.replace('az://','https://').split(';')[0]
36
37 az_sas = d.getVar('AZ_SAS')
38 if az_sas and az_sas not in ud.url:
39 if not az_sas.startswith('?'):
40 raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
41 ud.url += az_sas
42
43 return Wget.checkstatus(self, fetch, ud, d, try_again)
44
45 # Override download method, include retries
46 def download(self, ud, d, retries=3):
47 """Fetch urls"""
48
49 # If were reaching the account transaction limit we might be refused a connection,
50 # retrying allows us to avoid false negatives since the limit changes over time
51 fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
52
53 # We need to provide a localpath to avoid wget using the SAS
54 # ud.localfile either has the downloadfilename or ud.path
55 localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
56 bb.utils.mkdirhier(os.path.dirname(localpath))
57 fetchcmd += " -O %s" % shlex.quote(localpath)
58
59
60 if ud.user and ud.pswd:
61 fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
62
63 # Check if a Shared Access Signature was given and use it
64 az_sas = d.getVar('AZ_SAS')
65
66 if az_sas:
67 if not az_sas.startswith('?'):
68 raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
69 azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
70 else:
71 azuri = '%s%s%s' % ('https://', ud.host, ud.path)
72
73 dldir = d.getVar("DL_DIR")
74 if os.path.exists(ud.localpath):
75 # file exists, but we didnt complete it.. trying again.
76 fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
77 else:
78 fetchcmd += " -P %s '%s'" % (dldir, azuri)
79
80 try:
81 self._runwget(ud, d, fetchcmd, False)
82 except FetchError as e:
83 # Azure fails on handshake sometimes when using wget after some stress, producing a
84 # FetchError from the fetcher, if the artifact exists retyring should succeed
85 if 'Unable to establish SSL connection' in str(e):
86 logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
87 self.download(ud, d, retries -1)
88
89 # Sanity check since wget can pretend it succeed when it didn't
90 # Also, this used to happen if sourceforge sent us to the mirror page
91 if not os.path.exists(ud.localpath):
92 raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
93
94 if os.path.getsize(ud.localpath) == 0:
95 os.remove(ud.localpath)
96 raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
97
98 return True
diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
deleted file mode 100644
index fc558f50b0..0000000000
--- a/bitbake/lib/bb/fetch2/bzr.py
+++ /dev/null
@@ -1,128 +0,0 @@
1"""
2BitBake 'Fetch' implementation for bzr.
3
4"""
5
6# Copyright (C) 2007 Ross Burton
7# Copyright (C) 2007 Richard Purdie
8#
9# Classes for obtaining upstream sources for the
10# BitBake build tools.
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# SPDX-License-Identifier: GPL-2.0-only
14#
15
16import os
17import bb
18from bb.fetch2 import FetchMethod
19from bb.fetch2 import FetchError
20from bb.fetch2 import runfetchcmd
21from bb.fetch2 import logger
22
23class Bzr(FetchMethod):
24 def supports(self, ud, d):
25 return ud.type in ['bzr']
26
27 def urldata_init(self, ud, d):
28 """
29 init bzr specific variable within url data
30 """
31 # Create paths to bzr checkouts
32 bzrdir = d.getVar("BZRDIR") or (d.getVar("DL_DIR") + "/bzr")
33 relpath = self._strip_leading_slashes(ud.path)
34 ud.pkgdir = os.path.join(bzrdir, ud.host, relpath)
35
36 ud.setup_revisions(d)
37
38 if not ud.revision:
39 ud.revision = self.latest_revision(ud, d)
40
41 ud.localfile = d.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision))
42
43 def _buildbzrcommand(self, ud, d, command):
44 """
45 Build up an bzr commandline based on ud
46 command is "fetch", "update", "revno"
47 """
48
49 basecmd = d.getVar("FETCHCMD_bzr") or "/usr/bin/env bzr"
50
51 proto = ud.parm.get('protocol', 'http')
52
53 bzrroot = ud.host + ud.path
54
55 options = []
56
57 if command == "revno":
58 bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
59 else:
60 if ud.revision:
61 options.append("-r %s" % ud.revision)
62
63 if command == "fetch":
64 bzrcmd = "%s branch %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
65 elif command == "update":
66 bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options))
67 else:
68 raise FetchError("Invalid bzr command %s" % command, ud.url)
69
70 return bzrcmd
71
72 def download(self, ud, d):
73 """Fetch url"""
74
75 if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
76 bzrcmd = self._buildbzrcommand(ud, d, "update")
77 logger.debug("BZR Update %s", ud.url)
78 bb.fetch2.check_network_access(d, bzrcmd, ud.url)
79 runfetchcmd(bzrcmd, d, workdir=os.path.join(ud.pkgdir, os.path.basename(ud.path)))
80 else:
81 bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
82 bzrcmd = self._buildbzrcommand(ud, d, "fetch")
83 bb.fetch2.check_network_access(d, bzrcmd, ud.url)
84 logger.debug("BZR Checkout %s", ud.url)
85 bb.utils.mkdirhier(ud.pkgdir)
86 logger.debug("Running %s", bzrcmd)
87 runfetchcmd(bzrcmd, d, workdir=ud.pkgdir)
88
89 scmdata = ud.parm.get("scmdata", "")
90 if scmdata == "keep":
91 tar_flags = ""
92 else:
93 tar_flags = "--exclude='.bzr' --exclude='.bzrtags'"
94
95 # tar them up to a defined filename
96 runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)),
97 d, cleanup=[ud.localpath], workdir=ud.pkgdir)
98
99 def supports_srcrev(self):
100 return True
101
102 def _revision_key(self, ud, d, name):
103 """
104 Return a unique key for the url
105 """
106 return "bzr:" + ud.pkgdir
107
108 def _latest_revision(self, ud, d, name):
109 """
110 Return the latest upstream revision number
111 """
112 logger.debug2("BZR fetcher hitting network for %s", ud.url)
113
114 bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url)
115
116 output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True)
117
118 return output.strip()
119
120 def sortable_revision(self, ud, d, name):
121 """
122 Return a sortable revision number which in our case is the revision number
123 """
124
125 return False, self._build_revision(ud, d)
126
127 def _build_revision(self, ud, d):
128 return ud.revision
diff --git a/bitbake/lib/bb/fetch2/clearcase.py b/bitbake/lib/bb/fetch2/clearcase.py
deleted file mode 100644
index 17500daf95..0000000000
--- a/bitbake/lib/bb/fetch2/clearcase.py
+++ /dev/null
@@ -1,245 +0,0 @@
1"""
2BitBake 'Fetch' clearcase implementation
3
4The clearcase fetcher is used to retrieve files from a ClearCase repository.
5
6Usage in the recipe:
7
8 SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
9 SRCREV = "EXAMPLE_CLEARCASE_TAG"
10 PV = "${@d.getVar("SRCREV", False).replace("/", "+")}"
11
12The fetcher uses the rcleartool or cleartool remote client, depending on which one is available.
13
14Supported SRC_URI options are:
15
16- vob
17 (required) The name of the clearcase VOB (with prepending "/")
18
19- module
20 The module in the selected VOB (with prepending "/")
21
22 The module and vob parameters are combined to create
23 the following load rule in the view config spec:
24 load <vob><module>
25
26- proto
27 http or https
28
29Related variables:
30
31 CCASE_CUSTOM_CONFIG_SPEC
32 Write a config spec to this variable in your recipe to use it instead
33 of the default config spec generated by this fetcher.
34 Please note that the SRCREV loses its functionality if you specify
35 this variable. SRCREV is still used to label the archive after a fetch,
36 but it doesn't define what's fetched.
37
38User credentials:
39 cleartool:
40 The login of cleartool is handled by the system. No special steps needed.
41
42 rcleartool:
43 In order to use rcleartool with authenticated users an `rcleartool login` is
44 necessary before using the fetcher.
45"""
46# Copyright (C) 2014 Siemens AG
47#
48# SPDX-License-Identifier: GPL-2.0-only
49#
50
51import os
52import shutil
53import bb
54from bb.fetch2 import FetchMethod
55from bb.fetch2 import FetchError
56from bb.fetch2 import MissingParameterError
57from bb.fetch2 import ParameterError
58from bb.fetch2 import runfetchcmd
59from bb.fetch2 import logger
60
61class ClearCase(FetchMethod):
62 """Class to fetch urls via 'clearcase'"""
63 def init(self, d):
64 pass
65
66 def supports(self, ud, d):
67 """
68 Check to see if a given url can be fetched with Clearcase.
69 """
70 return ud.type in ['ccrc']
71
72 def debug(self, msg):
73 logger.debug("ClearCase: %s", msg)
74
75 def urldata_init(self, ud, d):
76 """
77 init ClearCase specific variable within url data
78 """
79 ud.proto = "https"
80 if 'protocol' in ud.parm:
81 ud.proto = ud.parm['protocol']
82 if not ud.proto in ('http', 'https'):
83 raise ParameterError("Invalid protocol type", ud.url)
84
85 ud.vob = ''
86 if 'vob' in ud.parm:
87 ud.vob = ud.parm['vob']
88 else:
89 msg = ud.url+": vob must be defined so the fetcher knows what to get."
90 raise MissingParameterError('vob', msg)
91
92 if 'module' in ud.parm:
93 ud.module = ud.parm['module']
94 else:
95 ud.module = ""
96
97 ud.basecmd = d.getVar("FETCHCMD_ccrc") or "/usr/bin/env cleartool || rcleartool"
98
99 if d.getVar("SRCREV") == "INVALID":
100 raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
101
102 ud.label = d.getVar("SRCREV", False)
103 ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC")
104
105 ud.server = "%s://%s%s" % (ud.proto, ud.host, ud.path)
106
107 ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""),
108 ud.module.replace("/", "."),
109 ud.label.replace("/", "."))
110
111 ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME"))
112 ud.csname = "%s-config-spec" % (ud.identifier)
113 ud.ccasedir = os.path.join(d.getVar("DL_DIR"), ud.type)
114 ud.viewdir = os.path.join(ud.ccasedir, ud.viewname)
115 ud.configspecfile = os.path.join(ud.ccasedir, ud.csname)
116 ud.localfile = "%s.tar.gz" % (ud.identifier)
117
118 self.debug("host = %s" % ud.host)
119 self.debug("path = %s" % ud.path)
120 self.debug("server = %s" % ud.server)
121 self.debug("proto = %s" % ud.proto)
122 self.debug("type = %s" % ud.type)
123 self.debug("vob = %s" % ud.vob)
124 self.debug("module = %s" % ud.module)
125 self.debug("basecmd = %s" % ud.basecmd)
126 self.debug("label = %s" % ud.label)
127 self.debug("ccasedir = %s" % ud.ccasedir)
128 self.debug("viewdir = %s" % ud.viewdir)
129 self.debug("viewname = %s" % ud.viewname)
130 self.debug("configspecfile = %s" % ud.configspecfile)
131 self.debug("localfile = %s" % ud.localfile)
132
133 def _build_ccase_command(self, ud, command):
134 """
135 Build up a commandline based on ud
136 command is: mkview, setcs, rmview
137 """
138 options = []
139
140 if "rcleartool" in ud.basecmd:
141 options.append("-server %s" % ud.server)
142
143 basecmd = "%s %s" % (ud.basecmd, command)
144
145 if command == 'mkview':
146 if not "rcleartool" in ud.basecmd:
147 # Cleartool needs a -snapshot view
148 options.append("-snapshot")
149 options.append("-tag %s" % ud.viewname)
150 options.append(ud.viewdir)
151
152 elif command == 'rmview':
153 options.append("-force")
154 options.append("%s" % ud.viewdir)
155
156 elif command == 'setcs':
157 options.append("-overwrite")
158 options.append(ud.configspecfile)
159
160 else:
161 raise FetchError("Invalid ccase command %s" % command)
162
163 ccasecmd = "%s %s" % (basecmd, " ".join(options))
164 self.debug("ccasecmd = %s" % ccasecmd)
165 return ccasecmd
166
167 def _write_configspec(self, ud, d):
168 """
169 Create config spec file (ud.configspecfile) for ccase view
170 """
171 config_spec = ""
172 custom_config_spec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", d)
173 if custom_config_spec is not None:
174 for line in custom_config_spec.split("\\n"):
175 config_spec += line+"\n"
176 bb.warn("A custom config spec has been set, SRCREV is only relevant for the tarball name.")
177 else:
178 config_spec += "element * CHECKEDOUT\n"
179 config_spec += "element * %s\n" % ud.label
180 config_spec += "load %s%s\n" % (ud.vob, ud.module)
181
182 logger.info("Using config spec: \n%s" % config_spec)
183
184 with open(ud.configspecfile, 'w') as f:
185 f.write(config_spec)
186
187 def _remove_view(self, ud, d):
188 if os.path.exists(ud.viewdir):
189 cmd = self._build_ccase_command(ud, 'rmview');
190 logger.info("cleaning up [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
191 bb.fetch2.check_network_access(d, cmd, ud.url)
192 output = runfetchcmd(cmd, d, workdir=ud.ccasedir)
193 logger.info("rmview output: %s", output)
194
195 def need_update(self, ud, d):
196 if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec):
197 ud.identifier += "-%s" % d.getVar("DATETIME")
198 return True
199 if os.path.exists(ud.localpath):
200 return False
201 return True
202
203 def supports_srcrev(self):
204 return True
205
206 def sortable_revision(self, ud, d, name):
207 return False, ud.identifier
208
209 def download(self, ud, d):
210 """Fetch url"""
211
212 # Make a fresh view
213 bb.utils.mkdirhier(ud.ccasedir)
214 self._write_configspec(ud, d)
215 cmd = self._build_ccase_command(ud, 'mkview')
216 logger.info("creating view [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
217 bb.fetch2.check_network_access(d, cmd, ud.url)
218 try:
219 runfetchcmd(cmd, d)
220 except FetchError as e:
221 if "CRCLI2008E" in e.msg:
222 raise FetchError("%s\n%s\n" % (e.msg, "Call `rcleartool login` in your console to authenticate to the clearcase server before running bitbake."))
223 else:
224 raise e
225
226 # Set configspec: Setting the configspec effectively fetches the files as defined in the configspec
227 cmd = self._build_ccase_command(ud, 'setcs');
228 logger.info("fetching data [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
229 bb.fetch2.check_network_access(d, cmd, ud.url)
230 output = runfetchcmd(cmd, d, workdir=ud.viewdir)
231 logger.info("%s", output)
232
233 # Copy the configspec to the viewdir so we have it in our source tarball later
234 shutil.copyfile(ud.configspecfile, os.path.join(ud.viewdir, ud.csname))
235
236 # Clean clearcase meta-data before tar
237
238 runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath], workdir = ud.viewdir)
239
240 # Clean up so we can create a new view next time
241 self.clean(ud, d);
242
243 def clean(self, ud, d):
244 self._remove_view(ud, d)
245 bb.utils.remove(ud.configspecfile)
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
deleted file mode 100644
index e611736f06..0000000000
--- a/bitbake/lib/bb/fetch2/crate.py
+++ /dev/null
@@ -1,150 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementation for crates.io
5"""
6
7# Copyright (C) 2016 Doug Goldstein
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11# Based on functions from the base bb module, Copyright 2003 Holger Schurig
12
13import hashlib
14import json
15import os
16import subprocess
17import bb
18from bb.fetch2 import logger, subprocess_setup, UnpackError
19from bb.fetch2.wget import Wget
20
21
22class Crate(Wget):
23
24 """Class to fetch crates via wget"""
25
26 def _cargo_bitbake_path(self, rootdir):
27 return os.path.join(rootdir, "cargo_home", "bitbake")
28
29 def supports(self, ud, d):
30 """
31 Check to see if a given url is for this fetcher
32 """
33 return ud.type in ['crate']
34
35 def recommends_checksum(self, urldata):
36 return True
37
38 def urldata_init(self, ud, d):
39 """
40 Sets up to download the respective crate from crates.io
41 """
42
43 if ud.type == 'crate':
44 self._crate_urldata_init(ud, d)
45
46 super(Crate, self).urldata_init(ud, d)
47
48 def _crate_urldata_init(self, ud, d):
49 """
50 Sets up the download for a crate
51 """
52
53 # URL syntax is: crate://NAME/VERSION
54 # break the URL apart by /
55 parts = ud.url.split('/')
56 if len(parts) < 5:
57 raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
58
59 # version is expected to be the last token
60 # but ignore possible url parameters which will be used
61 # by the top fetcher class
62 version = parts[-1].split(";")[0]
63 # second to last field is name
64 name = parts[-2]
65 # host (this is to allow custom crate registries to be specified
66 host = '/'.join(parts[2:-2])
67
68 # if using upstream just fix it up nicely
69 if host == 'crates.io':
70 host = 'crates.io/api/v1/crates'
71
72 ud.url = "https://%s/%s/%s/download" % (host, name, version)
73 ud.versionsurl = "https://%s/%s/versions" % (host, name)
74 ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
75 if 'name' not in ud.parm:
76 ud.parm['name'] = '%s-%s' % (name, version)
77
78 logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
79
80 def unpack(self, ud, rootdir, d):
81 """
82 Uses the crate to build the necessary paths for cargo to utilize it
83 """
84 if ud.type == 'crate':
85 return self._crate_unpack(ud, rootdir, d)
86 else:
87 super(Crate, self).unpack(ud, rootdir, d)
88
89 def _crate_unpack(self, ud, rootdir, d):
90 """
91 Unpacks a crate
92 """
93 thefile = ud.localpath
94
95 # possible metadata we need to write out
96 metadata = {}
97
98 # change to the rootdir to unpack but save the old working dir
99 save_cwd = os.getcwd()
100 os.chdir(rootdir)
101
102 bp = d.getVar('BP')
103 if bp == ud.parm.get('name'):
104 cmd = "tar -xz --no-same-owner -f %s" % thefile
105 ud.unpack_tracer.unpack("crate-extract", rootdir)
106 else:
107 cargo_bitbake = self._cargo_bitbake_path(rootdir)
108 ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
109
110 cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
111
112 # ensure we've got these paths made
113 bb.utils.mkdirhier(cargo_bitbake)
114
115 # generate metadata necessary
116 with open(thefile, 'rb') as f:
117 # get the SHA256 of the original tarball
118 tarhash = hashlib.sha256(f.read()).hexdigest()
119
120 metadata['files'] = {}
121 metadata['package'] = tarhash
122
123 path = d.getVar('PATH')
124 if path:
125 cmd = "PATH=\"%s\" %s" % (path, cmd)
126 bb.note("Unpacking %s to %s/" % (thefile, os.getcwd()))
127
128 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
129
130 os.chdir(save_cwd)
131
132 if ret != 0:
133 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
134
135 # if we have metadata to write out..
136 if len(metadata) > 0:
137 cratepath = os.path.splitext(os.path.basename(thefile))[0]
138 bbpath = self._cargo_bitbake_path(rootdir)
139 mdfile = '.cargo-checksum.json'
140 mdpath = os.path.join(bbpath, cratepath, mdfile)
141 with open(mdpath, "w") as f:
142 json.dump(metadata, f)
143
144 def latest_versionstring(self, ud, d):
145 from functools import cmp_to_key
146 json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
147 versions = [(0, i["num"], "") for i in json_data["versions"]]
148 versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
149
150 return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py
deleted file mode 100644
index 01de5ff4ca..0000000000
--- a/bitbake/lib/bb/fetch2/cvs.py
+++ /dev/null
@@ -1,157 +0,0 @@
1"""
2BitBake 'Fetch' implementations
3
4Classes for obtaining upstream sources for the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14#
15
16import os
17import bb
18from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger
19from bb.fetch2 import runfetchcmd
20
21class Cvs(FetchMethod):
22 """
23 Class to fetch a module or modules from cvs repositories
24 """
25 def supports(self, ud, d):
26 """
27 Check to see if a given url can be fetched with cvs.
28 """
29 return ud.type in ['cvs']
30
31 def urldata_init(self, ud, d):
32 if not "module" in ud.parm:
33 raise MissingParameterError("module", ud.url)
34 ud.module = ud.parm["module"]
35
36 ud.tag = ud.parm.get('tag', "")
37
38 # Override the default date in certain cases
39 if 'date' in ud.parm:
40 ud.date = ud.parm['date']
41 elif ud.tag:
42 ud.date = ""
43
44 norecurse = ''
45 if 'norecurse' in ud.parm:
46 norecurse = '_norecurse'
47
48 fullpath = ''
49 if 'fullpath' in ud.parm:
50 fullpath = '_fullpath'
51
52 ud.localfile = d.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath))
53
54 pkg = d.getVar('PN')
55 cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
56 ud.pkgdir = os.path.join(cvsdir, pkg)
57
58 def need_update(self, ud, d):
59 if (ud.date == "now"):
60 return True
61 if not os.path.exists(ud.localpath):
62 return True
63 return False
64
65 def download(self, ud, d):
66
67 method = ud.parm.get('method', 'pserver')
68 localdir = ud.parm.get('localdir', ud.module)
69 cvs_port = ud.parm.get('port', '')
70
71 cvs_rsh = None
72 if method == "ext":
73 if "rsh" in ud.parm:
74 cvs_rsh = ud.parm["rsh"]
75
76 if method == "dir":
77 cvsroot = ud.path
78 else:
79 cvsroot = ":" + method
80 cvsproxyhost = d.getVar('CVS_PROXY_HOST')
81 if cvsproxyhost:
82 cvsroot += ";proxy=" + cvsproxyhost
83 cvsproxyport = d.getVar('CVS_PROXY_PORT')
84 if cvsproxyport:
85 cvsroot += ";proxyport=" + cvsproxyport
86 cvsroot += ":" + ud.user
87 if ud.pswd:
88 cvsroot += ":" + ud.pswd
89 cvsroot += "@" + ud.host + ":" + cvs_port + ud.path
90
91 options = []
92 if 'norecurse' in ud.parm:
93 options.append("-l")
94 if ud.date:
95 # treat YYYYMMDDHHMM specially for CVS
96 if len(ud.date) == 12:
97 options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12]))
98 else:
99 options.append("-D \"%s UTC\"" % ud.date)
100 if ud.tag:
101 options.append("-r %s" % ud.tag)
102
103 cvsbasecmd = d.getVar("FETCHCMD_cvs") or "/usr/bin/env cvs"
104 cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
105 cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
106
107 if cvs_rsh:
108 cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
109 cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
110
111 # create module directory
112 logger.debug2("Fetch: checking for module directory")
113 moddir = os.path.join(ud.pkgdir, localdir)
114 workdir = None
115 if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
116 logger.info("Update " + ud.url)
117 bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url)
118 # update sources there
119 workdir = moddir
120 cmd = cvsupdatecmd
121 else:
122 logger.info("Fetch " + ud.url)
123 # check out sources there
124 bb.utils.mkdirhier(ud.pkgdir)
125 workdir = ud.pkgdir
126 logger.debug("Running %s", cvscmd)
127 bb.fetch2.check_network_access(d, cvscmd, ud.url)
128 cmd = cvscmd
129
130 runfetchcmd(cmd, d, cleanup=[moddir], workdir=workdir)
131
132 if not os.access(moddir, os.R_OK):
133 raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url)
134
135 scmdata = ud.parm.get("scmdata", "")
136 if scmdata == "keep":
137 tar_flags = ""
138 else:
139 tar_flags = "--exclude='CVS'"
140
141 # tar them up to a defined filename
142 workdir = None
143 if 'fullpath' in ud.parm:
144 workdir = ud.pkgdir
145 cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)
146 else:
147 workdir = os.path.dirname(os.path.realpath(moddir))
148 cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))
149
150 runfetchcmd(cmd, d, cleanup=[ud.localpath], workdir=workdir)
151
152 def clean(self, ud, d):
153 """ Clean CVS Files and tarballs """
154
155 bb.utils.remove(ud.pkgdir, True)
156 bb.utils.remove(ud.localpath)
157
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
deleted file mode 100644
index 86546d40bf..0000000000
--- a/bitbake/lib/bb/fetch2/gcp.py
+++ /dev/null
@@ -1,102 +0,0 @@
1"""
2BitBake 'Fetch' implementation for Google Cloup Platform Storage.
3
4Class for fetching files from Google Cloud Storage using the
5Google Cloud Storage Python Client. The GCS Python Client must
6be correctly installed, configured and authenticated prior to use.
7Additionally, gsutil must also be installed.
8
9"""
10
11# Copyright (C) 2023, Snap Inc.
12#
13# Based in part on bb.fetch2.s3:
14# Copyright (C) 2017 Andre McCurdy
15#
16# SPDX-License-Identifier: GPL-2.0-only
17#
18# Based on functions from the base bb module, Copyright 2003 Holger Schurig
19
20import os
21import bb
22import urllib.parse, urllib.error
23from bb.fetch2 import FetchMethod
24from bb.fetch2 import FetchError
25from bb.fetch2 import logger
26
27class GCP(FetchMethod):
28 """
29 Class to fetch urls via GCP's Python API.
30 """
31 def __init__(self):
32 self.gcp_client = None
33
34 def supports(self, ud, d):
35 """
36 Check to see if a given url can be fetched with GCP.
37 """
38 return ud.type in ['gs']
39
40 def recommends_checksum(self, urldata):
41 return True
42
43 def urldata_init(self, ud, d):
44 if 'downloadfilename' in ud.parm:
45 ud.basename = ud.parm['downloadfilename']
46 else:
47 ud.basename = os.path.basename(ud.path)
48
49 ud.localfile = ud.basename
50
51 def get_gcp_client(self):
52 from google.cloud import storage
53 self.gcp_client = storage.Client(project=None)
54
55 def download(self, ud, d):
56 """
57 Fetch urls using the GCP API.
58 Assumes localpath was called first.
59 """
60 from google.api_core.exceptions import NotFound
61 logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
62 if self.gcp_client is None:
63 self.get_gcp_client()
64
65 bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}")
66
67 # Path sometimes has leading slash, so strip it
68 path = ud.path.lstrip("/")
69 blob = self.gcp_client.bucket(ud.host).blob(path)
70 try:
71 blob.download_to_filename(ud.localpath)
72 except NotFound:
73 raise FetchError("The GCP API threw a NotFound exception")
74
75 # Additional sanity checks copied from the wget class (although there
76 # are no known issues which mean these are required, treat the GCP API
77 # tool with a little healthy suspicion).
78 if not os.path.exists(ud.localpath):
79 raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
80
81 if os.path.getsize(ud.localpath) == 0:
82 os.remove(ud.localpath)
83 raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
84
85 return True
86
87 def checkstatus(self, fetch, ud, d):
88 """
89 Check the status of a URL.
90 """
91 logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
92 if self.gcp_client is None:
93 self.get_gcp_client()
94
95 bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}")
96
97 # Path sometimes has leading slash, so strip it
98 path = ud.path.lstrip("/")
99 if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
100 raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
101 else:
102 return True
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
deleted file mode 100644
index 0fcdb19df1..0000000000
--- a/bitbake/lib/bb/fetch2/git.py
+++ /dev/null
@@ -1,1010 +0,0 @@
1"""
2BitBake 'Fetch' git implementation
3
4git fetcher support the SRC_URI with format of:
5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
6
7Supported SRC_URI options are:
8
9- branch
10 The git branch to retrieve from. The default is "master"
11
12- tag
13 The git tag to retrieve. The default is "master"
14
15- protocol
16 The method to use to access the repository. Common options are "git",
17 "http", "https", "file", "ssh" and "rsync". The default is "git".
18
19- rebaseable
20 rebaseable indicates that the upstream git repo may rebase in the future,
21 and current revision may disappear from upstream repo. This option will
22 remind fetcher to preserve local cache carefully for future use.
23 The default value is "0", set rebaseable=1 for rebaseable git repo.
24
25- nocheckout
26 Don't checkout source code when unpacking. set this option for the recipe
27 who has its own routine to checkout code.
28 The default is "0", set nocheckout=1 if needed.
29
30- bareclone
31 Create a bare clone of the source code and don't checkout the source code
32 when unpacking. Set this option for the recipe who has its own routine to
33 checkout code and tracking branch requirements.
34 The default is "0", set bareclone=1 if needed.
35
36- nobranch
37 Don't check the SHA validation for branch. set this option for the recipe
38 referring to commit which is valid in any namespace (branch, tag, ...)
39 instead of branch.
40 The default is "0", set nobranch=1 if needed.
41
42- subpath
43 Limit the checkout to a specific subpath of the tree.
44 By default, checkout the whole tree, set subpath=<path> if needed
45
46- destsuffix
47 The name of the path in which to place the checkout.
48 By default, the path is git/, set destsuffix=<suffix> if needed
49
50- usehead
51 For local git:// urls to use the current branch HEAD as the revision for use with
52 AUTOREV. Implies nobranch.
53
54- lfs
55 Enable the checkout to use LFS for large files. This will download all LFS files
56 in the download step, as the unpack step does not have network access.
57 The default is "1", set lfs=0 to skip.
58
59"""
60
61# Copyright (C) 2005 Richard Purdie
62#
63# SPDX-License-Identifier: GPL-2.0-only
64#
65
66import collections
67import errno
68import fnmatch
69import os
70import re
71import shlex
72import shutil
73import subprocess
74import tempfile
75import urllib
76import bb
77import bb.progress
78from contextlib import contextmanager
79from bb.fetch2 import FetchMethod
80from bb.fetch2 import runfetchcmd
81from bb.fetch2 import logger
82from bb.fetch2 import trusted_network
83
84
85sha1_re = re.compile(r'^[0-9a-f]{40}$')
86slash_re = re.compile(r"/+")
87
88class GitProgressHandler(bb.progress.LineFilterProgressHandler):
89 """Extract progress information from git output"""
90 def __init__(self, d):
91 self._buffer = ''
92 self._count = 0
93 super(GitProgressHandler, self).__init__(d)
94 # Send an initial progress event so the bar gets shown
95 self._fire_progress(-1)
96
97 def write(self, string):
98 self._buffer += string
99 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
100 stage_weights = [0.2, 0.05, 0.5, 0.25]
101 stagenum = 0
102 for i, stage in reversed(list(enumerate(stages))):
103 if stage in self._buffer:
104 stagenum = i
105 self._buffer = ''
106 break
107 self._status = stages[stagenum]
108 percs = re.findall(r'(\d+)%', string)
109 if percs:
110 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
111 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
112 if rates:
113 rate = rates[-1]
114 else:
115 rate = None
116 self.update(progress, rate)
117 else:
118 if stagenum == 0:
119 percs = re.findall(r': (\d+)', string)
120 if percs:
121 count = int(percs[-1])
122 if count > self._count:
123 self._count = count
124 self._fire_progress(-count)
125 super(GitProgressHandler, self).write(string)
126
127
128class Git(FetchMethod):
129 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
130 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
131
132 """Class to fetch a module or modules from git repositories"""
133 def init(self, d):
134 pass
135
136 def supports(self, ud, d):
137 """
138 Check to see if a given url can be fetched with git.
139 """
140 return ud.type in ['git']
141
142 def supports_checksum(self, urldata):
143 return False
144
145 def cleanup_upon_failure(self):
146 return False
147
148 def urldata_init(self, ud, d):
149 """
150 init git specific variable within url data
151 so that the git method like latest_revision() can work
152 """
153 if 'protocol' in ud.parm:
154 ud.proto = ud.parm['protocol']
155 elif not ud.host:
156 ud.proto = 'file'
157 else:
158 ud.proto = "git"
159 if ud.host == "github.com" and ud.proto == "git":
160 # github stopped supporting git protocol
161 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
162 ud.proto = "https"
163 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
164
165 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
166 raise bb.fetch2.ParameterError(f"Invalid protocol type: '{ud.proto}'", ud.url)
167
168 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
169
170 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
171
172 ud.nobranch = ud.parm.get("nobranch","0") == "1"
173
174 # usehead implies nobranch
175 ud.usehead = ud.parm.get("usehead","0") == "1"
176 if ud.usehead:
177 if ud.proto != "file":
178 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
179 ud.nobranch = 1
180
181 # bareclone implies nocheckout
182 ud.bareclone = ud.parm.get("bareclone","0") == "1"
183 if ud.bareclone:
184 ud.nocheckout = 1
185
186 ud.unresolvedrev = ""
187 ud.branch = ud.parm.get("branch", "")
188 if not ud.branch and not ud.nobranch:
189 raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
190
191 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
192
193 ud.cloneflags = "-n"
194 if not ud.noshared:
195 ud.cloneflags += " -s"
196 if ud.bareclone:
197 ud.cloneflags += " --mirror"
198
199 ud.shallow_skip_fast = False
200 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
201 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
202 if 'tag' in ud.parm:
203 ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag'])
204
205 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
206 if depth_default is not None:
207 try:
208 depth_default = int(depth_default or 0)
209 except ValueError:
210 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
211 else:
212 if depth_default < 0:
213 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
214 else:
215 depth_default = 1
216 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
217
218 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
219 ud.shallow_revs = []
220
221 ud.unresolvedrev = ud.branch
222
223 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
224 if shallow_depth is not None:
225 try:
226 shallow_depth = int(shallow_depth or 0)
227 except ValueError:
228 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
229 else:
230 if shallow_depth < 0:
231 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
232 ud.shallow_depths[ud.name] = shallow_depth
233
234 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
235 if revs is not None:
236 ud.shallow_revs.extend(revs.split())
237 elif revs_default is not None:
238 ud.shallow_revs.extend(revs_default.split())
239
240 if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
241 # Shallow disabled for this URL
242 ud.shallow = False
243
244 if ud.usehead:
245 # When usehead is set let's associate 'HEAD' with the unresolved
246 # rev of this repository. This will get resolved into a revision
247 # later. If an actual revision happens to have also been provided
248 # then this setting will be overridden.
249 ud.unresolvedrev = 'HEAD'
250
251 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
252
253 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
254 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
255 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
256
257 ud.setup_revisions(d)
258
259 # Ensure any revision that doesn't look like a SHA-1 is translated into one
260 if not sha1_re.match(ud.revision or ''):
261 if ud.revision:
262 ud.unresolvedrev = ud.revision
263 ud.revision = self.latest_revision(ud, d, ud.name)
264
265 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
266 if gitsrcname.startswith('.'):
267 gitsrcname = gitsrcname[1:]
268
269 # For a rebaseable git repo, it is necessary to keep a mirror tar ball
270 # per revision, so that even if the revision disappears from the
271 # upstream repo in the future, the mirror will remain intact and still
272 # contain the revision
273 if ud.rebaseable:
274 gitsrcname = gitsrcname + '_' + ud.revision
275
276 dl_dir = d.getVar("DL_DIR")
277 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
278 ud.clonedir = os.path.join(gitdir, gitsrcname)
279 ud.localfile = ud.clonedir
280
281 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
282 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
283 ud.mirrortarballs = [mirrortarball]
284 if ud.shallow:
285 tarballname = gitsrcname
286 if ud.bareclone:
287 tarballname = "%s_bare" % tarballname
288
289 if ud.shallow_revs:
290 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
291
292 tarballname = "%s_%s" % (tarballname, ud.revision[:7])
293 depth = ud.shallow_depths[ud.name]
294 if depth:
295 tarballname = "%s-%s" % (tarballname, depth)
296
297 shallow_refs = []
298 if not ud.nobranch:
299 shallow_refs.append(ud.branch)
300 if ud.shallow_extra_refs:
301 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
302 if shallow_refs:
303 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
304
305 fetcher = self.__class__.__name__.lower()
306 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
307 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
308 ud.mirrortarballs.insert(0, ud.shallowtarball)
309
310 def localpath(self, ud, d):
311 return ud.clonedir
312
313 def need_update(self, ud, d):
314 return self.clonedir_need_update(ud, d) \
315 or self.shallow_tarball_need_update(ud) \
316 or self.tarball_need_update(ud) \
317 or self.lfs_need_update(ud, d)
318
319 def clonedir_need_update(self, ud, d):
320 if not os.path.exists(ud.clonedir):
321 return True
322 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
323 return True
324 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
325 return True
326 if 'tag' in ud.parm and not self._contains_ref(ud, d, ud.name, ud.clonedir, tag=True):
327 return True
328 return False
329
330 def lfs_need_update(self, ud, d):
331 if not self._need_lfs(ud):
332 return False
333
334 if self.clonedir_need_update(ud, d):
335 return True
336
337 if not self._lfs_objects_downloaded(ud, d, ud.clonedir):
338 return True
339 return False
340
341 def clonedir_need_shallow_revs(self, ud, d):
342 for rev in ud.shallow_revs:
343 try:
344 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
345 except bb.fetch2.FetchError:
346 return rev
347 return None
348
349 def shallow_tarball_need_update(self, ud):
350 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
351
352 def tarball_need_update(self, ud):
353 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
354
355 def update_mirror_links(self, ud, origud):
356 super().update_mirror_links(ud, origud)
357 # When using shallow mode, add a symlink to the original fullshallow
358 # path to ensure a valid symlink even in the `PREMIRRORS` case
359 if ud.shallow and not os.path.exists(origud.fullshallow):
360 self.ensure_symlink(ud.localpath, origud.fullshallow)
361
362 def try_premirror(self, ud, d):
363 # If we don't do this, updating an existing checkout with only premirrors
364 # is not possible
365 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
366 return True
367 # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
368 # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
369 # we need to try premirrors first as using upstream is destined to fail.
370 if not trusted_network(d, ud.url):
371 return True
372 # the following check is to ensure incremental fetch in downloads, this is
373 # because the premirror might be old and does not contain the new rev required,
374 # and this will cause a total removal and new clone. So if we can reach to
375 # network, we prefer upstream over premirror, though the premirror might contain
376 # the new rev.
377 if os.path.exists(ud.clonedir):
378 return False
379 return True
380
381 def download(self, ud, d):
382 """Fetch url"""
383
384 # A current clone is preferred to either tarball, a shallow tarball is
385 # preferred to an out of date clone, and a missing clone will use
386 # either tarball.
387 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
388 ud.localpath = ud.fullshallow
389 return
390 elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
391 if not os.path.exists(ud.clonedir):
392 bb.utils.mkdirhier(ud.clonedir)
393 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
394 else:
395 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
396 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
397 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
398 if 'mirror' in output:
399 runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
400 runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
401 fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd)
402 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
403 repourl = self._get_repo_url(ud)
404
405 needs_clone = False
406 if os.path.exists(ud.clonedir):
407 # The directory may exist, but not be the top level of a bare git
408 # repository in which case it needs to be deleted and re-cloned.
409 try:
410 # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
411 output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
412 toplevel = output.rstrip()
413
414 if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
415 logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
416 needs_clone = True
417 except bb.fetch2.FetchError as e:
418 logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
419 needs_clone = True
420 except FileNotFoundError as e:
421 logger.warning("%s", e)
422 needs_clone = True
423
424 if needs_clone:
425 shutil.rmtree(ud.clonedir)
426 else:
427 needs_clone = True
428
429 # If the repo still doesn't exist, fallback to cloning it
430 if needs_clone:
431 # We do this since git will use a "-l" option automatically for local urls where possible,
432 # but it doesn't work when git/objects is a symlink, only works when it is a directory.
433 if repourl.startswith("file://"):
434 repourl_path = repourl[7:]
435 objects = os.path.join(repourl_path, 'objects')
436 if os.path.isdir(objects) and not os.path.islink(objects):
437 repourl = repourl_path
438 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
439 if ud.proto.lower() != 'file':
440 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
441 progresshandler = GitProgressHandler(d)
442
443 # Try creating a fast initial shallow clone
444 # Enabling ud.shallow_skip_fast will skip this
445 # If the Git error "Server does not allow request for unadvertised object"
446 # occurs, shallow_skip_fast is enabled automatically.
447 # This may happen if the Git server does not allow the request
448 # or if the Git client has issues with this functionality.
449 if ud.shallow and not ud.shallow_skip_fast:
450 try:
451 self.clone_shallow_with_tarball(ud, d)
452 # When the shallow clone has succeeded, use the shallow tarball
453 ud.localpath = ud.fullshallow
454 return
455 except:
456 logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
457
458 # When skipping fast initial shallow or the fast inital shallow clone failed:
459 # Try again with an initial regular clone
460 runfetchcmd(clone_cmd, d, log=progresshandler)
461
462 # Update the checkout if needed
463 if self.clonedir_need_update(ud, d):
464 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
465 if "origin" in output:
466 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
467
468 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
469
470 if ud.nobranch:
471 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
472 else:
473 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
474 if ud.proto.lower() != 'file':
475 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
476 progresshandler = GitProgressHandler(d)
477 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
478 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
479 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
480 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
481 try:
482 os.unlink(ud.fullmirror)
483 except OSError as exc:
484 if exc.errno != errno.ENOENT:
485 raise
486
487 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
488 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
489
490 if ud.shallow and ud.write_shallow_tarballs:
491 missing_rev = self.clonedir_need_shallow_revs(ud, d)
492 if missing_rev:
493 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
494
495 if self.lfs_need_update(ud, d):
496 self.lfs_fetch(ud, d, ud.clonedir, ud.revision)
497
498 def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
499 """Helper method for fetching Git LFS data"""
500 try:
501 if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision):
502 self._ensure_git_lfs(d, ud)
503
504 # Using worktree with the revision because .lfsconfig may exists
505 worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
506 runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
507 lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
508 runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
509 worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
510 runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
511 except:
512 logger.warning("Fetching LFS did not succeed.")
513
514 @contextmanager
515 def create_atomic(self, filename):
516 """Create as a temp file and move atomically into position to avoid races"""
517 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
518 try:
519 yield tfile
520 umask = os.umask(0o666)
521 os.umask(umask)
522 os.chmod(tfile, (0o666 & ~umask))
523 os.rename(tfile, filename)
524 finally:
525 os.close(fd)
526
527 def build_mirror_data(self, ud, d):
528 if ud.shallow and ud.write_shallow_tarballs:
529 if not os.path.exists(ud.fullshallow):
530 if os.path.islink(ud.fullshallow):
531 os.unlink(ud.fullshallow)
532 self.clone_shallow_with_tarball(ud, d)
533 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
534 if os.path.islink(ud.fullmirror):
535 os.unlink(ud.fullmirror)
536
537 logger.info("Creating tarball of git repository")
538 with self.create_atomic(ud.fullmirror) as tfile:
539 mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
540 quiet=True, workdir=ud.clonedir)
541 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
542 % (tfile, mtime), d, workdir=ud.clonedir)
543 runfetchcmd("touch %s.done" % ud.fullmirror, d)
544
545 def clone_shallow_with_tarball(self, ud, d):
546 ret = False
547 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
548 shallowclone = os.path.join(tempdir, 'git')
549 try:
550 try:
551 self.clone_shallow_local(ud, shallowclone, d)
552 except:
553 logger.warning("Fast shallow clone failed, try to skip fast mode now.")
554 bb.utils.remove(tempdir, recurse=True)
555 os.mkdir(tempdir)
556 ud.shallow_skip_fast = True
557 self.clone_shallow_local(ud, shallowclone, d)
558 logger.info("Creating tarball of git repository")
559 with self.create_atomic(ud.fullshallow) as tfile:
560 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
561 runfetchcmd("touch %s.done" % ud.fullshallow, d)
562 ret = True
563 finally:
564 bb.utils.remove(tempdir, recurse=True)
565
566 return ret
567
568 def clone_shallow_local(self, ud, dest, d):
569 """
570 Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
571 - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
572 - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
573 """
574
575 progresshandler = GitProgressHandler(d)
576 repourl = self._get_repo_url(ud)
577 bb.utils.mkdirhier(dest)
578 init_cmd = "%s init -q" % ud.basecmd
579 if ud.bareclone:
580 init_cmd += " --bare"
581 runfetchcmd(init_cmd, d, workdir=dest)
582 # Use repourl when creating a fast initial shallow clone
583 # Prefer already existing full bare clones if available
584 if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
585 remote = shlex.quote(repourl)
586 else:
587 remote = ud.clonedir
588 runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
589
590 # Check the histories which should be excluded
591 shallow_exclude = ''
592 for revision in ud.shallow_revs:
593 shallow_exclude += " --shallow-exclude=%s" % revision
594
595 revision = ud.revision
596 depth = ud.shallow_depths[ud.name]
597
598 # The --depth and --shallow-exclude can't be used together
599 if depth and shallow_exclude:
600 raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
601
602 # For nobranch, we need a ref, otherwise the commits will be
603 # removed, and for non-nobranch, we truncate the branch to our
604 # srcrev, to avoid keeping unnecessary history beyond that.
605 branch = ud.branch
606 if ud.nobranch:
607 ref = "refs/shallow/%s" % ud.name
608 elif ud.bareclone:
609 ref = "refs/heads/%s" % branch
610 else:
611 ref = "refs/remotes/origin/%s" % branch
612
613 fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
614 if depth:
615 fetch_cmd += " --depth %s" % depth
616
617 if shallow_exclude:
618 fetch_cmd += shallow_exclude
619
620 # Advertise the revision for lower version git such as 2.25.1:
621 # error: Server does not allow request for unadvertised object.
622 # The ud.clonedir is a local temporary dir, will be removed when
623 # fetch is done, so we can do anything on it.
624 adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
625 if ud.shallow_skip_fast:
626 runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
627
628 runfetchcmd(fetch_cmd, d, workdir=dest)
629 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
630 # Fetch Git LFS data
631 self.lfs_fetch(ud, d, dest, ud.revision)
632
633 # Apply extra ref wildcards
634 all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
635 d, workdir=dest).splitlines()
636 all_refs = []
637 for line in all_refs_remote:
638 all_refs.append(line.split()[-1])
639 extra_refs = []
640 for r in ud.shallow_extra_refs:
641 if not ud.bareclone:
642 r = r.replace('refs/heads/', 'refs/remotes/origin/')
643
644 if '*' in r:
645 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
646 extra_refs.extend(matches)
647 else:
648 extra_refs.append(r)
649
650 for ref in extra_refs:
651 ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '')
652 runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
653 revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
654 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
655
656 # The url is local ud.clonedir, set it to upstream one
657 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
658
659 def unpack(self, ud, destdir, d):
660 """ unpack the downloaded src to destdir"""
661
662 subdir = ud.parm.get("subdir")
663 subpath = ud.parm.get("subpath")
664 readpathspec = ""
665 def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/"
666
667 if subpath:
668 readpathspec = ":%s" % subpath
669 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
670
671 if subdir:
672 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
673 if os.path.isabs(subdir):
674 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
675 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
676 destdir = subdir
677 else:
678 destdir = os.path.join(destdir, subdir)
679 def_destsuffix = ""
680
681 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
682 destdir = ud.destdir = os.path.join(destdir, destsuffix)
683 if os.path.exists(destdir):
684 bb.utils.prunedir(destdir)
685 if not ud.bareclone:
686 ud.unpack_tracer.unpack("git", destdir)
687
688 need_lfs = self._need_lfs(ud)
689
690 if not need_lfs:
691 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
692
693 source_found = False
694 source_error = []
695
696 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
697 if clonedir_is_up_to_date:
698 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
699 source_found = True
700 else:
701 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
702
703 if not source_found:
704 if ud.shallow:
705 if os.path.exists(ud.fullshallow):
706 bb.utils.mkdirhier(destdir)
707 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
708 source_found = True
709 else:
710 source_error.append("shallow clone not available: " + ud.fullshallow)
711 else:
712 source_error.append("shallow clone not enabled")
713
714 if not source_found:
715 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
716
717 # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
718 # matches the revision
719 if 'tag' in ud.parm and sha1_re.match(ud.revision):
720 output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
721 output = output.strip()
722 if output != ud.revision:
723 # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list
724 # If it resolves to the same thing there isn't a problem.
725 output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
726 output2 = output2.strip()
727 if output != output2:
728 raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
729
730 repourl = self._get_repo_url(ud)
731 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
732
733 if self._contains_lfs(ud, d, destdir):
734 if not need_lfs:
735 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
736 else:
737 self._ensure_git_lfs(d, ud)
738
739 runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
740
741 if not ud.nocheckout:
742 if subpath:
743 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
744 workdir=destdir)
745 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
746 elif not ud.nobranch:
747 branchname = ud.branch
748 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
749 ud.revision), d, workdir=destdir)
750 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
751 branchname), d, workdir=destdir)
752 else:
753 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
754
755 return True
756
757 def clean(self, ud, d):
758 """ clean the git directory """
759
760 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
761 # The localpath is a symlink to clonedir when it is cloned from a
762 # mirror, so remove both of them.
763 if os.path.islink(ud.localpath):
764 clonedir = os.path.realpath(ud.localpath)
765 to_remove.append(clonedir)
766
767 # Remove shallow mirror tarball
768 if ud.shallow:
769 to_remove.append(ud.fullshallow)
770 to_remove.append(ud.fullshallow + ".done")
771
772 for r in to_remove:
773 if os.path.exists(r) or os.path.islink(r):
774 bb.note('Removing %s' % r)
775 bb.utils.remove(r, True)
776
777 def supports_srcrev(self):
778 return True
779
780 def _contains_ref(self, ud, d, name, wd, tag=False):
781 cmd = ""
782 git_ref_name = 'refs/tags/%s' % ud.parm['tag'] if tag else ud.revision
783
784 if ud.nobranch:
785 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
786 ud.basecmd, git_ref_name)
787 else:
788 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
789 ud.basecmd, git_ref_name, ud.branch)
790 try:
791 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
792 except bb.fetch2.FetchError:
793 return False
794 if len(output.split()) > 1:
795 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
796 return output.split()[0] != "0"
797
798 def _lfs_objects_downloaded(self, ud, d, wd):
799 """
800 Verifies whether the LFS objects for requested revisions have already been downloaded
801 """
802 # Bail out early if this repository doesn't use LFS
803 if not self._contains_lfs(ud, d, wd):
804 return True
805
806 self._ensure_git_lfs(d, ud)
807
808 # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
809 # existence.
810 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
811 cmd = "%s lfs ls-files -l %s" \
812 % (ud.basecmd, ud.revision)
813 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
814 # Do not do any further matching if no objects are managed by LFS
815 if not output:
816 return True
817
818 # Match all lines beginning with the hexadecimal OID
819 oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
820 for line in output.split("\n"):
821 oid = re.search(oid_regex, line)
822 if not oid:
823 bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
824 if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
825 return False
826
827 return True
828
829 def _need_lfs(self, ud):
830 return ud.parm.get("lfs", "1") == "1"
831
832 def _contains_lfs(self, ud, d, wd):
833 """
834 Check if the repository has 'lfs' (large file) content
835 """
836 cmd = "%s grep '^[^#].*lfs' %s:.gitattributes | wc -l" % (
837 ud.basecmd, ud.revision)
838
839 try:
840 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
841 if int(output) > 0:
842 return True
843 except (bb.fetch2.FetchError,ValueError):
844 pass
845 return False
846
847 def _ensure_git_lfs(self, d, ud):
848 """
849 Ensures that git-lfs is available, raising a FetchError if it isn't.
850 """
851 if shutil.which("git-lfs", path=d.getVar('PATH')) is None:
852 raise bb.fetch2.FetchError(
853 "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 "
854 "to ignore it)" % self._get_repo_url(ud))
855
856 def _get_repo_url(self, ud):
857 """
858 Return the repository URL
859 """
860 # Note that we do not support passwords directly in the git urls. There are several
861 # reasons. SRC_URI can be written out to things like buildhistory and people don't
862 # want to leak passwords like that. Its also all too easy to share metadata without
863 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
864 # alternatives so we will not take patches adding password support here.
865 if ud.user:
866 username = ud.user + '@'
867 else:
868 username = ""
869 return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
870
871 def _revision_key(self, ud, d, name):
872 """
873 Return a unique key for the url
874 """
875 # Collapse adjacent slashes
876 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
877
878 def _lsremote(self, ud, d, search):
879 """
880 Run git ls-remote with the specified search string
881 """
882 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
883 # and WORKDIR is in PATH (as a result of RSS), our call to
884 # runfetchcmd() exports PATH so this function will get called again (!)
885 # In this scenario the return call of the function isn't actually
886 # important - WORKDIR isn't needed in PATH to call git ls-remote
887 # anyway.
888 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
889 return ''
890 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
891 try:
892 repourl = self._get_repo_url(ud)
893 cmd = "%s ls-remote %s %s" % \
894 (ud.basecmd, shlex.quote(repourl), search)
895 if ud.proto.lower() != 'file':
896 bb.fetch2.check_network_access(d, cmd, repourl)
897 output = runfetchcmd(cmd, d, True)
898 if not output:
899 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
900 finally:
901 d.delVar('_BB_GIT_IN_LSREMOTE')
902 return output
903
904 def _latest_revision(self, ud, d, name):
905 """
906 Compute the HEAD revision for the url
907 """
908 if not d.getVar("__BBSRCREV_SEEN"):
909 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
910
911 # Ensure we mark as not cached
912 bb.fetch2.mark_recipe_nocache(d)
913
914 output = self._lsremote(ud, d, "")
915 # Tags of the form ^{} may not work, need to fallback to other form
916 if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
917 head = ud.unresolvedrev
918 tag = ud.unresolvedrev
919 else:
920 head = "refs/heads/%s" % ud.unresolvedrev
921 tag = "refs/tags/%s" % ud.unresolvedrev
922 for s in [head, tag + "^{}", tag]:
923 for l in output.strip().split('\n'):
924 sha1, ref = l.split()
925 if s == ref:
926 return sha1
927 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
928 (ud.unresolvedrev, ud.host+ud.path))
929
930 def latest_versionstring(self, ud, d):
931 """
932 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
933 by searching through the tags output of ls-remote, comparing
934 versions and returning the highest match.
935 """
936 pupver = ('', '')
937
938 try:
939 output = self._lsremote(ud, d, "refs/tags/*")
940 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
941 bb.note("Could not list remote: %s" % str(e))
942 return pupver
943
944 rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
945 pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
946 nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
947
948 verstring = ""
949 for line in output.split("\n"):
950 if not line:
951 break
952
953 m = rev_tag_re.match(line)
954 if not m:
955 continue
956
957 (revision, tag) = m.groups()
958
959 # Ignore non-released branches
960 if nonrel_re.search(tag):
961 continue
962
963 # search for version in the line
964 m = pver_re.search(tag)
965 if not m:
966 continue
967
968 pver = m.group('pver').replace("_", ".")
969
970 if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
971 continue
972
973 verstring = pver
974 pupver = (verstring, revision)
975
976 return pupver
977
978 def _build_revision(self, ud, d, name):
979 return ud.revision
980
981 def gitpkgv_revision(self, ud, d, name):
982 """
983 Return a sortable revision number by counting commits in the history
984 Based on gitpkgv.bblass in meta-openembedded
985 """
986 rev = ud.revision
987 localpath = ud.localpath
988 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
989 if not os.path.exists(localpath):
990 commits = None
991 else:
992 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
993 commits = bb.fetch2.runfetchcmd(
994 "git rev-list %s -- | wc -l" % shlex.quote(rev),
995 d, quiet=True).strip().lstrip('0')
996 if commits:
997 open(rev_file, "w").write("%d\n" % int(commits))
998 else:
999 commits = open(rev_file, "r").readline(128).strip()
1000 if commits:
1001 return False, "%s+%s" % (commits, rev[:7])
1002 else:
1003 return True, str(rev)
1004
1005 def checkstatus(self, fetch, ud, d):
1006 try:
1007 self._lsremote(ud, d, "")
1008 return True
1009 except bb.fetch2.FetchError:
1010 return False
diff --git a/bitbake/lib/bb/fetch2/gitannex.py b/bitbake/lib/bb/fetch2/gitannex.py
deleted file mode 100644
index 80a808d88f..0000000000
--- a/bitbake/lib/bb/fetch2/gitannex.py
+++ /dev/null
@@ -1,77 +0,0 @@
1"""
2BitBake 'Fetch' git annex implementation
3"""
4
5# Copyright (C) 2014 Otavio Salvador
6# Copyright (C) 2014 O.S. Systems Software LTDA.
7#
8# SPDX-License-Identifier: GPL-2.0-only
9#
10
11import bb
12from bb.fetch2.git import Git
13from bb.fetch2 import runfetchcmd
14
15class GitANNEX(Git):
16 def supports(self, ud, d):
17 """
18 Check to see if a given url can be fetched with git.
19 """
20 return ud.type in ['gitannex']
21
22 def urldata_init(self, ud, d):
23 super(GitANNEX, self).urldata_init(ud, d)
24 if ud.shallow:
25 ud.shallow_extra_refs += ['refs/heads/git-annex', 'refs/heads/synced/*']
26
27 def uses_annex(self, ud, d, wd):
28 for name in ud.names:
29 try:
30 runfetchcmd("%s rev-list git-annex" % (ud.basecmd), d, quiet=True, workdir=wd)
31 return True
32 except bb.fetch.FetchError:
33 pass
34
35 return False
36
37 def update_annex(self, ud, d, wd):
38 try:
39 runfetchcmd("%s annex get --all" % (ud.basecmd), d, quiet=True, workdir=wd)
40 except bb.fetch.FetchError:
41 return False
42 runfetchcmd("chmod u+w -R %s/annex" % (ud.clonedir), d, quiet=True, workdir=wd)
43
44 return True
45
46 def download(self, ud, d):
47 Git.download(self, ud, d)
48
49 if not ud.shallow or ud.localpath != ud.fullshallow:
50 if self.uses_annex(ud, d, ud.clonedir):
51 self.update_annex(ud, d, ud.clonedir)
52
53 def clone_shallow_local(self, ud, dest, d):
54 super(GitANNEX, self).clone_shallow_local(ud, dest, d)
55
56 try:
57 runfetchcmd("%s annex init" % ud.basecmd, d, workdir=dest)
58 except bb.fetch.FetchError:
59 pass
60
61 if self.uses_annex(ud, d, dest):
62 runfetchcmd("%s annex get" % ud.basecmd, d, workdir=dest)
63 runfetchcmd("chmod u+w -R %s/.git/annex" % (dest), d, quiet=True, workdir=dest)
64
65 def unpack(self, ud, destdir, d):
66 Git.unpack(self, ud, destdir, d)
67
68 try:
69 runfetchcmd("%s annex init" % (ud.basecmd), d, workdir=ud.destdir)
70 except bb.fetch.FetchError:
71 pass
72
73 annex = self.uses_annex(ud, d, ud.destdir)
74 if annex:
75 runfetchcmd("%s annex get" % (ud.basecmd), d, workdir=ud.destdir)
76 runfetchcmd("chmod u+w -R %s/.git/annex" % (ud.destdir), d, quiet=True, workdir=ud.destdir)
77
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
deleted file mode 100644
index 5869e1b99b..0000000000
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ /dev/null
@@ -1,278 +0,0 @@
1"""
2BitBake 'Fetch' git submodules implementation
3
4Inherits from and extends the Git fetcher to retrieve submodules of a git repository
5after cloning.
6
7SRC_URI = "gitsm://<see Git fetcher for syntax>"
8
9See the Git fetcher, git://, for usage documentation.
10
11NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
12
13"""
14
15# Copyright (C) 2013 Richard Purdie
16#
17# SPDX-License-Identifier: GPL-2.0-only
18#
19
20import os
21import bb
22import copy
23import shutil
24import tempfile
25from bb.fetch2.git import Git
26from bb.fetch2 import runfetchcmd
27from bb.fetch2 import logger
28from bb.fetch2 import Fetch
29
30class GitSM(Git):
31 def supports(self, ud, d):
32 """
33 Check to see if a given url can be fetched with git.
34 """
35 return ud.type in ['gitsm']
36
37 def process_submodules(self, ud, workdir, function, d):
38 """
39 Iterate over all of the submodules in this repository and execute
40 the 'function' for each of them.
41 """
42
43 submodules = []
44 paths = {}
45 revision = {}
46 uris = {}
47 subrevision = {}
48
49 def parse_gitmodules(gitmodules):
50 modules = {}
51 module = ""
52 for line in gitmodules.splitlines():
53 if line.startswith('[submodule'):
54 module = line.split('"')[1]
55 modules[module] = {}
56 elif module and line.strip().startswith('path'):
57 path = line.split('=')[1].strip()
58 modules[module]['path'] = path
59 elif module and line.strip().startswith('url'):
60 url = line.split('=')[1].strip()
61 modules[module]['url'] = url
62 return modules
63
64 # Collect the defined submodules, and their attributes
65 try:
66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
67 except:
68 # No submodules to update
69 gitmodules = ""
70
71 for m, md in parse_gitmodules(gitmodules).items():
72 try:
73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
74 except:
75 # If the command fails, we don't have a valid file to check. If it doesn't
76 # fail -- it still might be a failure, see next check...
77 module_hash = ""
78
79 if not module_hash:
80 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
81 continue
82
83 submodules.append(m)
84 paths[m] = md['path']
85 revision[m] = ud.revision
86 uris[m] = md['url']
87 subrevision[m] = module_hash.split()[2]
88
89 # Convert relative to absolute uri based on parent uri
90 if uris[m].startswith('..') or uris[m].startswith('./'):
91 newud = copy.copy(ud)
92 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
93 uris[m] = Git._get_repo_url(self, newud)
94
95 for module in submodules:
96 # Translate the module url into a SRC_URI
97
98 if "://" in uris[module]:
99 # Properly formated URL already
100 proto = uris[module].split(':', 1)[0]
101 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
102 else:
103 if ":" in uris[module]:
104 # Most likely an SSH style reference
105 proto = "ssh"
106 if ":/" in uris[module]:
107 # Absolute reference, easy to convert..
108 url = "gitsm://" + uris[module].replace(':/', '/', 1)
109 else:
110 # Relative reference, no way to know if this is right!
111 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
112 url = "gitsm://" + uris[module].replace(':', '/', 1)
113 else:
114 # This has to be a file reference
115 proto = "file"
116 url = "gitsm://" + uris[module]
117 if url.endswith("{}{}".format(ud.host, ud.path)):
118 raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
119 "Consider using git fetcher instead.")
120
121 url += ';protocol=%s' % proto
122 url += ";name=%s" % module
123 url += ";subpath=%s" % module
124 url += ";nobranch=1"
125 url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0")
126 # Note that adding "user=" here to give credentials to the
127 # submodule is not supported. Since using SRC_URI to give git://
128 # URL a password is not supported, one have to use one of the
129 # recommended way (eg. ~/.netrc or SSH config) which does specify
130 # the user (See comment in git.py).
131 # So, we will not take patches adding "user=" support here.
132
133 ld = d.createCopy()
134 # Not necessary to set SRC_URI, since we're passing the URI to
135 # Fetch.
136 #ld.setVar('SRC_URI', url)
137 ld.setVar('SRCREV_%s' % module, subrevision[module])
138
139 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
140 # error refer to 'multiple' repositories. Only the repository
141 # in the original SRC_URI actually matters...
142 ld.setVar('SRCPV', d.getVar('SRCPV'))
143 ld.setVar('SRCREV_FORMAT', module)
144
145 function(ud, url, module, paths[module], workdir, ld)
146
147 return submodules != []
148
149 def call_process_submodules(self, ud, d, extra_check, subfunc):
150 # If we're using a shallow mirror tarball it needs to be
151 # unpacked temporarily so that we can examine the .gitmodules file
152 # Unpack even when ud.clonedir is not available,
153 # which may occur during a fast shallow clone
154 unpack = extra_check or not os.path.exists(ud.clonedir)
155 if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
156 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
157 try:
158 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
159 self.process_submodules(ud, tmpdir, subfunc, d)
160 finally:
161 shutil.rmtree(tmpdir)
162 else:
163 self.process_submodules(ud, ud.clonedir, subfunc, d)
164
165 def need_update(self, ud, d):
166 if Git.need_update(self, ud, d):
167 return True
168
169 need_update_list = []
170 def need_update_submodule(ud, url, module, modpath, workdir, d):
171 url += ";bareclone=1;nobranch=1"
172
173 try:
174 newfetch = Fetch([url], d, cache=False)
175 new_ud = newfetch.ud[url]
176 if new_ud.method.need_update(new_ud, d):
177 need_update_list.append(modpath)
178 except Exception as e:
179 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
180 need_update_result = True
181
182 self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
183
184 if need_update_list:
185 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
186 return True
187
188 return False
189
190 def download(self, ud, d):
191 def download_submodule(ud, url, module, modpath, workdir, d):
192 url += ";bareclone=1;nobranch=1"
193
194 # Is the following still needed?
195 #url += ";nocheckout=1"
196
197 try:
198 newfetch = Fetch([url], d, cache=False)
199 newfetch.download()
200 except Exception as e:
201 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
202 raise
203
204 Git.download(self, ud, d)
205 self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
206
207 def unpack(self, ud, destdir, d):
208 def unpack_submodules(ud, url, module, modpath, workdir, d):
209 url += ";bareclone=1;nobranch=1"
210
211 # Figure out where we clone over the bare submodules...
212 if ud.bareclone:
213 repo_conf = ud.destdir
214 else:
215 repo_conf = os.path.join(ud.destdir, '.git')
216
217 try:
218 newfetch = Fetch([url], d, cache=False)
219 # modpath is needed by unpack tracer to calculate submodule
220 # checkout dir
221 new_ud = newfetch.ud[url]
222 new_ud.modpath = modpath
223 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
224 except Exception as e:
225 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
226 raise
227
228 local_path = newfetch.localpath(url)
229
230 # Correct the submodule references to the local download version...
231 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
232
233 if ud.shallow:
234 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
235
236 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
237 try:
238 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module))
239 except:
240 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module))
241 raise
242
243 Git.unpack(self, ud, destdir, d)
244
245 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
246
247 if not ud.bareclone and ret:
248 cmdprefix = ""
249 # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed.
250 if not self._need_lfs(ud):
251 cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 "
252 runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir)
253 def clean(self, ud, d):
254 def clean_submodule(ud, url, module, modpath, workdir, d):
255 url += ";bareclone=1;nobranch=1"
256 try:
257 newfetch = Fetch([url], d, cache=False)
258 newfetch.clean()
259 except Exception as e:
260 logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
261
262 self.call_process_submodules(ud, d, True, clean_submodule)
263
264 # Clean top git dir
265 Git.clean(self, ud, d)
266
267 def implicit_urldata(self, ud, d):
268 import subprocess
269
270 urldata = []
271 def add_submodule(ud, url, module, modpath, workdir, d):
272 url += ";bareclone=1;nobranch=1"
273 newfetch = Fetch([url], d, cache=False)
274 urldata.extend(newfetch.expanded_urldata())
275
276 self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
277
278 return urldata
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py
deleted file mode 100644
index 53c1d8d115..0000000000
--- a/bitbake/lib/bb/fetch2/gomod.py
+++ /dev/null
@@ -1,273 +0,0 @@
1"""
2BitBake 'Fetch' implementation for Go modules
3
4The gomod/gomodgit fetchers are used to download Go modules to the module cache
5from a module proxy or directly from a version control repository.
6
7Example SRC_URI:
8
9SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
10SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
11
12Required SRC_URI parameters:
13
14- version
15 The version of the module.
16
17Optional SRC_URI parameters:
18
19- mod
20 Fetch and unpack the go.mod file only instead of the complete module.
21 The go command may need to download go.mod files for many different modules
22 when computing the build list, and go.mod files are much smaller than
23 module zip files.
24 The default is "0", set mod=1 for the go.mod file only.
25
26- sha256sum
27 The checksum of the module zip file, or the go.mod file in case of fetching
28 only the go.mod file. Alternatively, set the SRC_URI varible flag for
29 "module@version.sha256sum".
30
31- protocol
32 The method used when fetching directly from a version control repository.
33 The default is "https" for git.
34
35- repo
36 The URL when fetching directly from a version control repository. Required
37 when the URL is different from the module path.
38
39- srcrev
40 The revision identifier used when fetching directly from a version control
41 repository. Alternatively, set the SRCREV varible for "module@version".
42
43- subdir
44 The module subdirectory when fetching directly from a version control
45 repository. Required when the module is not located in the root of the
46 repository.
47
48Related variables:
49
50- GO_MOD_PROXY
51 The module proxy used by the fetcher.
52
53- GO_MOD_CACHE_DIR
54 The directory where the module cache is located.
55 This must match the exported GOMODCACHE variable for the go command to find
56 the downloaded modules.
57
58See the Go modules reference, https://go.dev/ref/mod, for more information
59about the module cache, module proxies and version control systems.
60"""
61
62import hashlib
63import os
64import re
65import shutil
66import subprocess
67import zipfile
68
69import bb
70from bb.fetch2 import FetchError
71from bb.fetch2 import MissingParameterError
72from bb.fetch2 import runfetchcmd
73from bb.fetch2 import subprocess_setup
74from bb.fetch2.git import Git
75from bb.fetch2.wget import Wget
76
77
78def escape(path):
79 """Escape capital letters using exclamation points."""
80 return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
81
82
83class GoMod(Wget):
84 """Class to fetch Go modules from a Go module proxy via wget"""
85
86 def supports(self, ud, d):
87 """Check to see if a given URL is for this fetcher."""
88 return ud.type == 'gomod'
89
90 def urldata_init(self, ud, d):
91 """Set up to download the module from the module proxy.
92
93 Set up to download the module zip file to the module cache directory
94 and unpack the go.mod file (unless downloading only the go.mod file):
95
96 cache/download/<module>/@v/<version>.zip: The module zip file.
97 cache/download/<module>/@v/<version>.mod: The go.mod file.
98 """
99
100 proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
101 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
102
103 if 'version' not in ud.parm:
104 raise MissingParameterError('version', ud.url)
105
106 module = ud.host
107 if ud.path != '/':
108 module += ud.path
109 ud.parm['module'] = module
110 version = ud.parm['version']
111
112 # Set URL and filename for wget download
113 if ud.parm.get('mod', '0') == '1':
114 ext = '.mod'
115 else:
116 ext = '.zip'
117 path = escape(f"{module}/@v/{version}{ext}")
118 ud.url = bb.fetch2.encodeurl(
119 ('https', proxy, '/' + path, None, None, None))
120 ud.parm['downloadfilename'] = f"{module.replace('/', '.')}@{version}{ext}"
121
122 # Set name for checksum verification
123 ud.parm['name'] = f"{module}@{version}"
124
125 # Set path for unpack
126 ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path)
127
128 super().urldata_init(ud, d)
129
130 def unpack(self, ud, rootdir, d):
131 """Unpack the module in the module cache."""
132
133 # Unpack the module zip file or go.mod file
134 unpackpath = os.path.join(rootdir, ud.parm['unpackpath'])
135 unpackdir = os.path.dirname(unpackpath)
136 bb.utils.mkdirhier(unpackdir)
137 ud.unpack_tracer.unpack("file-copy", unpackdir)
138 cmd = f"cp {ud.localpath} {unpackpath}"
139 path = d.getVar('PATH')
140 if path:
141 cmd = f"PATH={path} {cmd}"
142 name = os.path.basename(unpackpath)
143 bb.note(f"Unpacking {name} to {unpackdir}/")
144 subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup)
145
146 if name.endswith('.zip'):
147 # Unpack the go.mod file from the zip file
148 module = ud.parm['module']
149 name = name.rsplit('.', 1)[0] + '.mod'
150 bb.note(f"Unpacking {name} to {unpackdir}/")
151 with zipfile.ZipFile(ud.localpath) as zf:
152 with open(os.path.join(unpackdir, name), mode='wb') as mf:
153 try:
154 f = module + '@' + ud.parm['version'] + '/go.mod'
155 shutil.copyfileobj(zf.open(f), mf)
156 except KeyError:
157 # If the module does not have a go.mod file, synthesize
158 # one containing only a module statement.
159 mf.write(f'module {module}\n'.encode())
160
161
162class GoModGit(Git):
163 """Class to fetch Go modules directly from a git repository"""
164
165 def supports(self, ud, d):
166 """Check to see if a given URL is for this fetcher."""
167 return ud.type == 'gomodgit'
168
169 def urldata_init(self, ud, d):
170 """Set up to download the module from the git repository.
171
172 Set up to download the git repository to the module cache directory and
173 unpack the module zip file and the go.mod file:
174
175 cache/vcs/<hash>: The bare git repository.
176 cache/download/<module>/@v/<version>.zip: The module zip file.
177 cache/download/<module>/@v/<version>.mod: The go.mod file.
178 """
179
180 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
181
182 if 'version' not in ud.parm:
183 raise MissingParameterError('version', ud.url)
184
185 module = ud.host
186 if ud.path != '/':
187 module += ud.path
188 ud.parm['module'] = module
189
190 # Set host, path and srcrev for git download
191 if 'repo' in ud.parm:
192 repo = ud.parm['repo']
193 idx = repo.find('/')
194 if idx != -1:
195 ud.host = repo[:idx]
196 ud.path = repo[idx:]
197 else:
198 ud.host = repo
199 ud.path = ''
200 if 'protocol' not in ud.parm:
201 ud.parm['protocol'] = 'https'
202 ud.name = f"{module}@{ud.parm['version']}"
203 srcrev = d.getVar('SRCREV_' + ud.name)
204 if srcrev:
205 if 'srcrev' not in ud.parm:
206 ud.parm['srcrev'] = srcrev
207 else:
208 if 'srcrev' in ud.parm:
209 d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
210 if 'branch' not in ud.parm:
211 ud.parm['nobranch'] = '1'
212
213 # Set subpath, subdir and bareclone for git unpack
214 if 'subdir' in ud.parm:
215 ud.parm['subpath'] = ud.parm['subdir']
216 key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
217 ud.parm['key'] = key
218 ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
219 hashlib.sha256(key).hexdigest())
220 ud.parm['bareclone'] = '1'
221
222 super().urldata_init(ud, d)
223
224 def unpack(self, ud, rootdir, d):
225 """Unpack the module in the module cache."""
226
227 # Unpack the bare git repository
228 super().unpack(ud, rootdir, d)
229
230 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
231
232 # Create the info file
233 module = ud.parm['module']
234 repodir = os.path.join(rootdir, ud.parm['subdir'])
235 with open(repodir + '.info', 'wb') as f:
236 f.write(ud.parm['key'])
237
238 # Unpack the go.mod file from the repository
239 unpackdir = os.path.join(rootdir, moddir, 'cache/download',
240 escape(module), '@v')
241 bb.utils.mkdirhier(unpackdir)
242 srcrev = ud.parm['srcrev']
243 version = ud.parm['version']
244 escaped_version = escape(version)
245 cmd = f"git ls-tree -r --name-only '{srcrev}'"
246 if 'subpath' in ud.parm:
247 cmd += f" '{ud.parm['subpath']}'"
248 files = runfetchcmd(cmd, d, workdir=repodir).split()
249 name = escaped_version + '.mod'
250 bb.note(f"Unpacking {name} to {unpackdir}/")
251 with open(os.path.join(unpackdir, name), mode='wb') as mf:
252 f = 'go.mod'
253 if 'subpath' in ud.parm:
254 f = os.path.join(ud.parm['subpath'], f)
255 if f in files:
256 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
257 subprocess.check_call(cmd, stdout=mf, cwd=repodir,
258 preexec_fn=subprocess_setup)
259 else:
260 # If the module does not have a go.mod file, synthesize one
261 # containing only a module statement.
262 mf.write(f'module {module}\n'.encode())
263
264 # Synthesize the module zip file from the repository
265 name = escaped_version + '.zip'
266 bb.note(f"Unpacking {name} to {unpackdir}/")
267 with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
268 prefix = module + '@' + version + '/'
269 for f in files:
270 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
271 data = subprocess.check_output(cmd, cwd=repodir,
272 preexec_fn=subprocess_setup)
273 zf.writestr(prefix + f, data)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
deleted file mode 100644
index cbff8c490c..0000000000
--- a/bitbake/lib/bb/fetch2/hg.py
+++ /dev/null
@@ -1,264 +0,0 @@
1"""
2BitBake 'Fetch' implementation for mercurial DRCS (hg).
3
4"""
5
6# Copyright (C) 2003, 2004 Chris Larson
7# Copyright (C) 2004 Marcin Juszkiewicz
8# Copyright (C) 2007 Robert Schuster
9#
10# SPDX-License-Identifier: GPL-2.0-only
11#
12# Based on functions from the base bb module, Copyright 2003 Holger Schurig
13#
14
15import os
16import bb
17import errno
18from bb.fetch2 import FetchMethod
19from bb.fetch2 import FetchError
20from bb.fetch2 import MissingParameterError
21from bb.fetch2 import runfetchcmd
22from bb.fetch2 import logger
23
24class Hg(FetchMethod):
25 """Class to fetch from mercurial repositories"""
26 def supports(self, ud, d):
27 """
28 Check to see if a given url can be fetched with mercurial.
29 """
30 return ud.type in ['hg']
31
32 def supports_checksum(self, urldata):
33 """
34 Don't require checksums for local archives created from
35 repository checkouts.
36 """
37 return False
38
39 def urldata_init(self, ud, d):
40 """
41 init hg specific variable within url data
42 """
43 if not "module" in ud.parm:
44 raise MissingParameterError('module', ud.url)
45
46 ud.module = ud.parm["module"]
47
48 if 'protocol' in ud.parm:
49 ud.proto = ud.parm['protocol']
50 elif not ud.host:
51 ud.proto = 'file'
52 else:
53 ud.proto = "hg"
54
55 # Create paths to mercurial checkouts
56 hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
57 ud.host, ud.path.replace('/', '.'))
58 mirrortarball = 'hg_%s.tar.gz' % hgsrcname
59 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
60 ud.mirrortarballs = [mirrortarball]
61
62 hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg")
63 ud.pkgdir = os.path.join(hgdir, hgsrcname)
64 ud.moddir = os.path.join(ud.pkgdir, ud.module)
65 ud.localfile = ud.moddir
66 ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
67
68 ud.setup_revisions(d)
69
70 if 'rev' in ud.parm:
71 ud.revision = ud.parm['rev']
72 elif not ud.revision:
73 ud.revision = self.latest_revision(ud, d)
74
75 ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
76
77 def need_update(self, ud, d):
78 revTag = ud.parm.get('rev', 'tip')
79 if revTag == "tip":
80 return True
81 if not os.path.exists(ud.localpath):
82 return True
83 return False
84
85 def try_premirror(self, ud, d):
86 # If we don't do this, updating an existing checkout with only premirrors
87 # is not possible
88 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
89 return True
90 if os.path.exists(ud.moddir):
91 return False
92 return True
93
94 def _buildhgcommand(self, ud, d, command):
95 """
96 Build up an hg commandline based on ud
97 command is "fetch", "update", "info"
98 """
99
100 proto = ud.parm.get('protocol', 'http')
101
102 host = ud.host
103 if proto == "file":
104 host = "/"
105 ud.host = "localhost"
106
107 if not ud.user:
108 hgroot = host + ud.path
109 else:
110 if ud.pswd:
111 hgroot = ud.user + ":" + ud.pswd + "@" + host + ud.path
112 else:
113 hgroot = ud.user + "@" + host + ud.path
114
115 if command == "info":
116 return "%s identify -i %s://%s/%s" % (ud.basecmd, proto, hgroot, ud.module)
117
118 options = [];
119
120 # Don't specify revision for the fetch; clone the entire repo.
121 # This avoids an issue if the specified revision is a tag, because
122 # the tag actually exists in the specified revision + 1, so it won't
123 # be available when used in any successive commands.
124 if ud.revision and command != "fetch":
125 options.append("-r %s" % ud.revision)
126
127 if command == "fetch":
128 if ud.user and ud.pswd:
129 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module)
130 else:
131 cmd = "%s clone %s %s://%s/%s %s" % (ud.basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
132 elif command == "pull":
133 # do not pass options list; limiting pull to rev causes the local
134 # repo not to contain it and immediately following "update" command
135 # will crash
136 if ud.user and ud.pswd:
137 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
138 else:
139 cmd = "%s pull" % (ud.basecmd)
140 elif command == "update" or command == "up":
141 if ud.user and ud.pswd:
142 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
143 else:
144 cmd = "%s update -C %s" % (ud.basecmd, " ".join(options))
145 else:
146 raise FetchError("Invalid hg command %s" % command, ud.url)
147
148 return cmd
149
150 def download(self, ud, d):
151 """Fetch url"""
152
153 logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
154
155 # If the checkout doesn't exist and the mirror tarball does, extract it
156 if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
157 bb.utils.mkdirhier(ud.pkgdir)
158 runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.pkgdir)
159
160 if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
161 # Found the source, check whether need pull
162 updatecmd = self._buildhgcommand(ud, d, "update")
163 logger.debug("Running %s", updatecmd)
164 try:
165 runfetchcmd(updatecmd, d, workdir=ud.moddir)
166 except bb.fetch2.FetchError:
167 # Runnning pull in the repo
168 pullcmd = self._buildhgcommand(ud, d, "pull")
169 logger.info("Pulling " + ud.url)
170 # update sources there
171 logger.debug("Running %s", pullcmd)
172 bb.fetch2.check_network_access(d, pullcmd, ud.url)
173 runfetchcmd(pullcmd, d, workdir=ud.moddir)
174 try:
175 os.unlink(ud.fullmirror)
176 except OSError as exc:
177 if exc.errno != errno.ENOENT:
178 raise
179
180 # No source found, clone it.
181 if not os.path.exists(ud.moddir):
182 fetchcmd = self._buildhgcommand(ud, d, "fetch")
183 logger.info("Fetch " + ud.url)
184 # check out sources there
185 bb.utils.mkdirhier(ud.pkgdir)
186 logger.debug("Running %s", fetchcmd)
187 bb.fetch2.check_network_access(d, fetchcmd, ud.url)
188 runfetchcmd(fetchcmd, d, workdir=ud.pkgdir)
189
190 # Even when we clone (fetch), we still need to update as hg's clone
191 # won't checkout the specified revision if its on a branch
192 updatecmd = self._buildhgcommand(ud, d, "update")
193 logger.debug("Running %s", updatecmd)
194 runfetchcmd(updatecmd, d, workdir=ud.moddir)
195
196 def clean(self, ud, d):
197 """ Clean the hg dir """
198
199 bb.utils.remove(ud.localpath, True)
200 bb.utils.remove(ud.fullmirror)
201 bb.utils.remove(ud.fullmirror + ".done")
202
203 def supports_srcrev(self):
204 return True
205
206 def _latest_revision(self, ud, d, name):
207 """
208 Compute tip revision for the url
209 """
210 bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"), ud.url)
211 output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
212 return output.strip()
213
214 def _build_revision(self, ud, d, name):
215 return ud.revision
216
217 def _revision_key(self, ud, d, name):
218 """
219 Return a unique key for the url
220 """
221 return "hg:" + ud.moddir
222
223 def build_mirror_data(self, ud, d):
224 # Generate a mirror tarball if needed
225 if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror):
226 # it's possible that this symlink points to read-only filesystem with PREMIRROR
227 if os.path.islink(ud.fullmirror):
228 os.unlink(ud.fullmirror)
229
230 logger.info("Creating tarball of hg repository")
231 runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d, workdir=ud.pkgdir)
232 runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.pkgdir)
233
234 def localpath(self, ud, d):
235 return ud.pkgdir
236
237 def unpack(self, ud, destdir, d):
238 """
239 Make a local clone or export for the url
240 """
241
242 revflag = "-r %s" % ud.revision
243 subdir = ud.parm.get("destsuffix", ud.module)
244 codir = "%s/%s" % (destdir, subdir)
245 ud.unpack_tracer.unpack("hg", codir)
246
247 scmdata = ud.parm.get("scmdata", "")
248 if scmdata != "nokeep":
249 proto = ud.parm.get('protocol', 'http')
250 if not os.access(os.path.join(codir, '.hg'), os.R_OK):
251 logger.debug2("Unpack: creating new hg repository in '" + codir + "'")
252 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
253 logger.debug2("Unpack: updating source in '" + codir + "'")
254 if ud.user and ud.pswd:
255 runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
256 else:
257 runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
258 if ud.user and ud.pswd:
259 runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir)
260 else:
261 runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
262 else:
263 logger.debug2("Unpack: extracting source to '" + codir + "'")
264 runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
deleted file mode 100644
index fda56a564e..0000000000
--- a/bitbake/lib/bb/fetch2/local.py
+++ /dev/null
@@ -1,91 +0,0 @@
1"""
2BitBake 'Fetch' implementations
3
4Classes for obtaining upstream sources for the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14#
15
16import os
17import urllib.request, urllib.parse, urllib.error
18import bb
19import bb.utils
20from bb.fetch2 import FetchMethod, FetchError, ParameterError
21from bb.fetch2 import logger
22
23class Local(FetchMethod):
24 def supports(self, urldata, d):
25 """
26 Check to see if a given url represents a local fetch.
27 """
28 return urldata.type in ['file']
29
30 def urldata_init(self, ud, d):
31 # We don't set localfile as for this fetcher the file is already local!
32 ud.basename = os.path.basename(ud.path)
33 ud.basepath = ud.path
34 ud.needdonestamp = False
35 if "*" in ud.path:
36 raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
37 return
38
39 def localpath(self, urldata, d):
40 """
41 Return the local filename of a given url assuming a successful fetch.
42 """
43 return self.localfile_searchpaths(urldata, d)[-1]
44
45 def localfile_searchpaths(self, urldata, d):
46 """
47 Return the local filename of a given url assuming a successful fetch.
48 """
49 searched = []
50 path = urldata.path
51 newpath = path
52 if path[0] == "/":
53 logger.debug2("Using absolute %s" % (path))
54 return [path]
55 filespath = d.getVar('FILESPATH')
56 if filespath:
57 logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":"))))
58 newpath, hist = bb.utils.which(filespath, path, history=True)
59 logger.debug2("Using %s for %s" % (newpath, path))
60 searched.extend(hist)
61 return searched
62
63 def need_update(self, ud, d):
64 if os.path.exists(ud.localpath):
65 return False
66 return True
67
68 def download(self, urldata, d):
69 """Fetch urls (no-op for Local method)"""
70 # no need to fetch local files, we'll deal with them in place.
71 if self.supports_checksum(urldata) and not os.path.exists(urldata.localpath):
72 locations = []
73 filespath = d.getVar('FILESPATH')
74 if filespath:
75 locations = filespath.split(":")
76 msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n " + "\n ".join(locations)
77 raise FetchError(msg)
78
79 return True
80
81 def checkstatus(self, fetch, urldata, d):
82 """
83 Check the status of the url
84 """
85 if os.path.exists(urldata.localpath):
86 return True
87 return False
88
89 def clean(self, urldata, d):
90 return
91
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
deleted file mode 100644
index e469d66768..0000000000
--- a/bitbake/lib/bb/fetch2/npm.py
+++ /dev/null
@@ -1,331 +0,0 @@
1# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5"""
6BitBake 'Fetch' npm implementation
7
8npm fetcher support the SRC_URI with format of:
9SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..."
10
11Supported SRC_URI options are:
12
13- package
14 The npm package name. This is a mandatory parameter.
15
16- version
17 The npm package version. This is a mandatory parameter.
18
19- downloadfilename
20 Specifies the filename used when storing the downloaded file.
21
22- destsuffix
23 Specifies the directory to use to unpack the package (default: npm).
24"""
25
26import base64
27import json
28import os
29import re
30import shlex
31import tempfile
32import bb
33from bb.fetch2 import Fetch
34from bb.fetch2 import FetchError
35from bb.fetch2 import FetchMethod
36from bb.fetch2 import MissingParameterError
37from bb.fetch2 import ParameterError
38from bb.fetch2 import URI
39from bb.fetch2 import check_network_access
40from bb.fetch2 import runfetchcmd
41from bb.utils import is_semver
42
43def npm_package(package):
44 """Convert the npm package name to remove unsupported character"""
45 # For scoped package names ('@user/package') the '/' is replaced by a '-'.
46 # This is similar to what 'npm pack' does, but 'npm pack' also strips the
47 # leading '@', which can lead to ambiguous package names.
48 name = re.sub("/", "-", package)
49 name = name.lower()
50 name = re.sub(r"[^\-a-z0-9@]", "", name)
51 name = name.strip("-")
52 return name
53
54
55def npm_filename(package, version):
56 """Get the filename of a npm package"""
57 return npm_package(package) + "-" + version + ".tgz"
58
59def npm_localfile(package, version=None):
60 """Get the local filename of a npm package"""
61 if version is not None:
62 filename = npm_filename(package, version)
63 else:
64 filename = package
65 return os.path.join("npm2", filename)
66
67def npm_integrity(integrity):
68 """
69 Get the checksum name and expected value from the subresource integrity
70 https://www.w3.org/TR/SRI/
71 """
72 algo, value = integrity.split("-", maxsplit=1)
73 return "%ssum" % algo, base64.b64decode(value).hex()
74
75def npm_unpack(tarball, destdir, d):
76 """Unpack a npm tarball"""
77 bb.utils.mkdirhier(destdir)
78 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
79 cmd += " --no-same-owner"
80 cmd += " --delay-directory-restore"
81 cmd += " --strip-components=1"
82 runfetchcmd(cmd, d, workdir=destdir)
83 runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
84
85class NpmEnvironment(object):
86 """
87 Using a npm config file seems more reliable than using cli arguments.
88 This class allows to create a controlled environment for npm commands.
89 """
90 def __init__(self, d, configs=[], npmrc=None):
91 self.d = d
92
93 self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
94
95 hn = self._home_npmrc(d)
96 if hn is not None:
97 with open(hn, 'r') as hnf:
98 self.user_config.write(hnf.read())
99
100 for key, value in configs:
101 self.user_config.write("%s=%s\n" % (key, value))
102
103 if npmrc:
104 self.global_config_name = npmrc
105 else:
106 self.global_config_name = "/dev/null"
107
108 def __del__(self):
109 if self.user_config:
110 self.user_config.close()
111
112 def _home_npmrc(self, d):
113 """Function to return user's HOME .npmrc file (or None if it doesn't exist)"""
114 home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc")
115 if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file):
116 bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\
117 f"npm fetcher will use {home_npmrc_file}")
118 return home_npmrc_file
119 return None
120
121 def run(self, cmd, args=None, configs=None, workdir=None):
122 """Run npm command in a controlled environment"""
123 with tempfile.TemporaryDirectory() as tmpdir:
124 d = bb.data.createCopy(self.d)
125 d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching
126 d.setVar("HOME", tmpdir)
127
128 if not workdir:
129 workdir = tmpdir
130
131 def _run(cmd):
132 cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
133 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
134 return runfetchcmd(cmd, d, workdir=workdir)
135
136 if configs:
137 bb.warn("Use of configs argument of NpmEnvironment.run() function"
138 " is deprecated. Please use args argument instead.")
139 for key, value in configs:
140 cmd += " --%s=%s" % (key, shlex.quote(value))
141
142 if args:
143 for key, value in args:
144 cmd += " --%s=%s" % (key, shlex.quote(value))
145
146 return _run(cmd)
147
148class Npm(FetchMethod):
149 """Class to fetch a package from a npm registry"""
150
151 def supports(self, ud, d):
152 """Check if a given url can be fetched with npm"""
153 return ud.type in ["npm"]
154
155 def urldata_init(self, ud, d):
156 """Init npm specific variables within url data"""
157 ud.package = None
158 ud.version = None
159 ud.registry = None
160
161 # Get the 'package' parameter
162 if "package" in ud.parm:
163 ud.package = ud.parm.get("package")
164
165 if not ud.package:
166 raise MissingParameterError("Parameter 'package' required", ud.url)
167
168 # Get the 'version' parameter
169 if "version" in ud.parm:
170 ud.version = ud.parm.get("version")
171
172 if not ud.version:
173 raise MissingParameterError("Parameter 'version' required", ud.url)
174
175 if not is_semver(ud.version) and not ud.version == "latest":
176 raise ParameterError("Invalid 'version' parameter", ud.url)
177
178 # Extract the 'registry' part of the url
179 ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
180
181 # Using the 'downloadfilename' parameter as local filename
182 # or the npm package name.
183 if "downloadfilename" in ud.parm:
184 ud.localfile = npm_localfile(ud.parm["downloadfilename"])
185 else:
186 ud.localfile = npm_localfile(ud.package, ud.version)
187
188 # Get the base 'npm' command
189 ud.basecmd = d.getVar("FETCHCMD_npm") or "npm"
190
191 # This fetcher resolves a URI from a npm package name and version and
192 # then forwards it to a proxy fetcher. A resolve file containing the
193 # resolved URI is created to avoid unwanted network access (if the file
194 # already exists). The management of the donestamp file, the lockfile
195 # and the checksums are forwarded to the proxy fetcher.
196 ud.proxy = None
197 ud.needdonestamp = False
198 ud.resolvefile = self.localpath(ud, d) + ".resolved"
199
200 def _resolve_proxy_url(self, ud, d):
201 def _npm_view():
202 args = []
203 args.append(("json", "true"))
204 args.append(("registry", ud.registry))
205 pkgver = shlex.quote(ud.package + "@" + ud.version)
206 cmd = ud.basecmd + " view %s" % pkgver
207 env = NpmEnvironment(d)
208 check_network_access(d, cmd, ud.registry)
209 view_string = env.run(cmd, args=args)
210
211 if not view_string:
212 raise FetchError("Unavailable package %s" % pkgver, ud.url)
213
214 try:
215 view = json.loads(view_string)
216
217 error = view.get("error")
218 if error is not None:
219 raise FetchError(error.get("summary"), ud.url)
220
221 if ud.version == "latest":
222 bb.warn("The npm package %s is using the latest " \
223 "version available. This could lead to " \
224 "non-reproducible builds." % pkgver)
225 elif ud.version != view.get("version"):
226 raise ParameterError("Invalid 'version' parameter", ud.url)
227
228 return view
229
230 except Exception as e:
231 raise FetchError("Invalid view from npm: %s" % str(e), ud.url)
232
233 def _get_url(view):
234 tarball_url = view.get("dist", {}).get("tarball")
235
236 if tarball_url is None:
237 raise FetchError("Invalid 'dist.tarball' in view", ud.url)
238
239 uri = URI(tarball_url)
240 uri.params["downloadfilename"] = ud.localfile
241
242 integrity = view.get("dist", {}).get("integrity")
243 shasum = view.get("dist", {}).get("shasum")
244
245 if integrity is not None:
246 checksum_name, checksum_expected = npm_integrity(integrity)
247 uri.params[checksum_name] = checksum_expected
248 elif shasum is not None:
249 uri.params["sha1sum"] = shasum
250 else:
251 raise FetchError("Invalid 'dist.integrity' in view", ud.url)
252
253 return str(uri)
254
255 url = _get_url(_npm_view())
256
257 bb.utils.mkdirhier(os.path.dirname(ud.resolvefile))
258 with open(ud.resolvefile, "w") as f:
259 f.write(url)
260
261 def _setup_proxy(self, ud, d):
262 if ud.proxy is None:
263 if not os.path.exists(ud.resolvefile):
264 self._resolve_proxy_url(ud, d)
265
266 with open(ud.resolvefile, "r") as f:
267 url = f.read()
268
269 # Avoid conflicts between the environment data and:
270 # - the proxy url checksum
271 data = bb.data.createCopy(d)
272 data.delVarFlags("SRC_URI")
273 ud.proxy = Fetch([url], data)
274
275 def _get_proxy_method(self, ud, d):
276 self._setup_proxy(ud, d)
277 proxy_url = ud.proxy.urls[0]
278 proxy_ud = ud.proxy.ud[proxy_url]
279 proxy_d = ud.proxy.d
280 proxy_ud.setup_localpath(proxy_d)
281 return proxy_ud.method, proxy_ud, proxy_d
282
283 def verify_donestamp(self, ud, d):
284 """Verify the donestamp file"""
285 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
286 return proxy_m.verify_donestamp(proxy_ud, proxy_d)
287
288 def update_donestamp(self, ud, d):
289 """Update the donestamp file"""
290 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
291 proxy_m.update_donestamp(proxy_ud, proxy_d)
292
293 def need_update(self, ud, d):
294 """Force a fetch, even if localpath exists ?"""
295 if not os.path.exists(ud.resolvefile):
296 return True
297 if ud.version == "latest":
298 return True
299 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
300 return proxy_m.need_update(proxy_ud, proxy_d)
301
302 def try_mirrors(self, fetch, ud, d, mirrors):
303 """Try to use a mirror"""
304 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
305 return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors)
306
307 def download(self, ud, d):
308 """Fetch url"""
309 self._setup_proxy(ud, d)
310 ud.proxy.download()
311
312 def unpack(self, ud, rootdir, d):
313 """Unpack the downloaded archive"""
314 destsuffix = ud.parm.get("destsuffix", "npm")
315 destdir = os.path.join(rootdir, destsuffix)
316 npm_unpack(ud.localpath, destdir, d)
317 ud.unpack_tracer.unpack("npm", destdir)
318
319 def clean(self, ud, d):
320 """Clean any existing full or partial download"""
321 if os.path.exists(ud.resolvefile):
322 self._setup_proxy(ud, d)
323 ud.proxy.clean()
324 bb.utils.remove(ud.resolvefile)
325
326 def done(self, ud, d):
327 """Is the download done ?"""
328 if not os.path.exists(ud.resolvefile):
329 return False
330 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
331 return proxy_m.done(proxy_ud, proxy_d)
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
deleted file mode 100644
index 2f9599ee9e..0000000000
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ /dev/null
@@ -1,291 +0,0 @@
1# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5"""
6BitBake 'Fetch' npm shrinkwrap implementation
7
8npm fetcher support the SRC_URI with format of:
9SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
10
11Supported SRC_URI options are:
12
13- dev
14 Set to 1 to also install devDependencies.
15
16- destsuffix
17 Specifies the directory to use to unpack the dependencies (default: ${S}).
18"""
19
20import json
21import os
22import re
23import bb
24from bb.fetch2 import Fetch
25from bb.fetch2 import FetchMethod
26from bb.fetch2 import ParameterError
27from bb.fetch2 import runfetchcmd
28from bb.fetch2 import URI
29from bb.fetch2.npm import npm_integrity
30from bb.fetch2.npm import npm_localfile
31from bb.fetch2.npm import npm_unpack
32from bb.utils import is_semver
33from bb.utils import lockfile
34from bb.utils import unlockfile
35
36def foreach_dependencies(shrinkwrap, callback=None, dev=False):
37 """
38 Run a callback for each dependencies of a shrinkwrap file.
39 The callback is using the format:
40 callback(name, data, location)
41 with:
42 name = the package name (string)
43 data = the package data (dictionary)
44 location = the location of the package (string)
45 """
46 packages = shrinkwrap.get("packages")
47 if not packages:
48 raise FetchError("Invalid shrinkwrap file format")
49
50 for location, data in packages.items():
51 # Skip empty main and local link target packages
52 if not location.startswith('node_modules/'):
53 continue
54 elif not dev and data.get("dev", False):
55 continue
56 elif data.get("inBundle", False):
57 continue
58 name = location.split('node_modules/')[-1]
59 callback(name, data, location)
60
61class NpmShrinkWrap(FetchMethod):
62 """Class to fetch all package from a shrinkwrap file"""
63
64 def supports(self, ud, d):
65 """Check if a given url can be fetched with npmsw"""
66 return ud.type in ["npmsw"]
67
68 def urldata_init(self, ud, d):
69 """Init npmsw specific variables within url data"""
70
71 # Get the 'shrinkwrap' parameter
72 ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
73
74 # Get the 'dev' parameter
75 ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
76
77 # Resolve the dependencies
78 ud.deps = []
79
80 def _resolve_dependency(name, params, destsuffix):
81 url = None
82 localpath = None
83 extrapaths = []
84 unpack = True
85
86 integrity = params.get("integrity")
87 resolved = params.get("resolved")
88 version = params.get("version")
89 link = params.get("link", False)
90
91 # Handle link sources
92 if link:
93 localpath = resolved
94 unpack = False
95
96 # Handle registry sources
97 elif version and is_semver(version) and integrity:
98 # Handle duplicate dependencies without url
99 if not resolved:
100 return
101
102 localfile = npm_localfile(name, version)
103
104 uri = URI(resolved)
105 uri.params["downloadfilename"] = localfile
106
107 checksum_name, checksum_expected = npm_integrity(integrity)
108 uri.params[checksum_name] = checksum_expected
109
110 url = str(uri)
111
112 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
113
114 # Create a resolve file to mimic the npm fetcher and allow
115 # re-usability of the downloaded file.
116 resolvefile = localpath + ".resolved"
117
118 bb.utils.mkdirhier(os.path.dirname(resolvefile))
119 with open(resolvefile, "w") as f:
120 f.write(url)
121
122 extrapaths.append(resolvefile)
123
124 # Handle http tarball sources
125 elif resolved.startswith("http") and integrity:
126 localfile = npm_localfile(os.path.basename(resolved))
127
128 uri = URI(resolved)
129 uri.params["downloadfilename"] = localfile
130
131 checksum_name, checksum_expected = npm_integrity(integrity)
132 uri.params[checksum_name] = checksum_expected
133
134 url = str(uri)
135
136 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
137
138 # Handle local tarball sources
139 elif resolved.startswith("file"):
140 localpath = resolved[5:]
141
142 # Handle git sources
143 elif resolved.startswith("git"):
144 regex = re.compile(r"""
145 ^
146 git\+
147 (?P<protocol>[a-z]+)
148 ://
149 (?P<url>[^#]+)
150 \#
151 (?P<rev>[0-9a-f]+)
152 $
153 """, re.VERBOSE)
154
155 match = regex.match(resolved)
156 if not match:
157 raise ParameterError("Invalid git url: %s" % resolved, ud.url)
158
159 groups = match.groupdict()
160
161 uri = URI("git://" + str(groups["url"]))
162 uri.params["protocol"] = str(groups["protocol"])
163 uri.params["rev"] = str(groups["rev"])
164 uri.params["nobranch"] = "1"
165 uri.params["destsuffix"] = destsuffix
166
167 url = str(uri)
168
169 else:
170 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
171
172 # name is needed by unpack tracer for module mapping
173 ud.deps.append({
174 "name": name,
175 "url": url,
176 "localpath": localpath,
177 "extrapaths": extrapaths,
178 "destsuffix": destsuffix,
179 "unpack": unpack,
180 })
181
182 try:
183 with open(ud.shrinkwrap_file, "r") as f:
184 shrinkwrap = json.load(f)
185 except Exception as e:
186 raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
187
188 foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
189
190 # Avoid conflicts between the environment data and:
191 # - the proxy url revision
192 # - the proxy url checksum
193 data = bb.data.createCopy(d)
194 data.delVar("SRCREV")
195 data.delVarFlags("SRC_URI")
196
197 # This fetcher resolves multiple URIs from a shrinkwrap file and then
198 # forwards it to a proxy fetcher. The management of the donestamp file,
199 # the lockfile and the checksums are forwarded to the proxy fetcher.
200 shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
201 if shrinkwrap_urls:
202 ud.proxy = Fetch(shrinkwrap_urls, data)
203 ud.needdonestamp = False
204
205 @staticmethod
206 def _foreach_proxy_method(ud, handle):
207 returns = []
208 #Check if there are dependencies before try to fetch them
209 if len(ud.deps) > 0:
210 for proxy_url in ud.proxy.urls:
211 proxy_ud = ud.proxy.ud[proxy_url]
212 proxy_d = ud.proxy.d
213 proxy_ud.setup_localpath(proxy_d)
214 lf = lockfile(proxy_ud.lockfile)
215 returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
216 unlockfile(lf)
217 return returns
218
219 def verify_donestamp(self, ud, d):
220 """Verify the donestamp file"""
221 def _handle(m, ud, d):
222 return m.verify_donestamp(ud, d)
223 return all(self._foreach_proxy_method(ud, _handle))
224
225 def update_donestamp(self, ud, d):
226 """Update the donestamp file"""
227 def _handle(m, ud, d):
228 m.update_donestamp(ud, d)
229 self._foreach_proxy_method(ud, _handle)
230
231 def need_update(self, ud, d):
232 """Force a fetch, even if localpath exists ?"""
233 def _handle(m, ud, d):
234 return m.need_update(ud, d)
235 return all(self._foreach_proxy_method(ud, _handle))
236
237 def try_mirrors(self, fetch, ud, d, mirrors):
238 """Try to use a mirror"""
239 def _handle(m, ud, d):
240 return m.try_mirrors(fetch, ud, d, mirrors)
241 return all(self._foreach_proxy_method(ud, _handle))
242
243 def download(self, ud, d):
244 """Fetch url"""
245 ud.proxy.download()
246
247 def unpack(self, ud, rootdir, d):
248 """Unpack the downloaded dependencies"""
249 destdir = rootdir
250 destsuffix = ud.parm.get("destsuffix")
251 if destsuffix:
252 destdir = os.path.join(rootdir, destsuffix)
253 ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
254
255 bb.utils.mkdirhier(destdir)
256 bb.utils.copyfile(ud.shrinkwrap_file,
257 os.path.join(destdir, "npm-shrinkwrap.json"))
258
259 auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
260 manual = [dep for dep in ud.deps if dep["localpath"]]
261
262 if auto:
263 ud.proxy.unpack(destdir, auto)
264
265 for dep in manual:
266 depdestdir = os.path.join(destdir, dep["destsuffix"])
267 if dep["url"]:
268 npm_unpack(dep["localpath"], depdestdir, d)
269 else:
270 depsrcdir= os.path.join(destdir, dep["localpath"])
271 if dep["unpack"]:
272 npm_unpack(depsrcdir, depdestdir, d)
273 else:
274 bb.utils.mkdirhier(depdestdir)
275 cmd = 'cp -fpPRH "%s/." .' % (depsrcdir)
276 runfetchcmd(cmd, d, workdir=depdestdir)
277
278 def clean(self, ud, d):
279 """Clean any existing full or partial download"""
280 ud.proxy.clean()
281
282 # Clean extra files
283 for dep in ud.deps:
284 for path in dep["extrapaths"]:
285 bb.utils.remove(path)
286
287 def done(self, ud, d):
288 """Is the download done ?"""
289 def _handle(m, ud, d):
290 return m.done(ud, d)
291 return all(self._foreach_proxy_method(ud, _handle))
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
deleted file mode 100644
index 495ac8a30a..0000000000
--- a/bitbake/lib/bb/fetch2/osc.py
+++ /dev/null
@@ -1,165 +0,0 @@
1#
2# Copyright BitBake Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""
7Bitbake "Fetch" implementation for osc (Opensuse build service client).
8Based on the svn "Fetch" implementation.
9
10"""
11
12import logging
13import os
14import re
15import bb
16from bb.fetch2 import FetchMethod
17from bb.fetch2 import FetchError
18from bb.fetch2 import MissingParameterError
19from bb.fetch2 import runfetchcmd
20
21logger = logging.getLogger(__name__)
22
23class Osc(FetchMethod):
24 """Class to fetch a module or modules from Opensuse build server
25 repositories."""
26
27 def supports(self, ud, d):
28 """
29 Check to see if a given url can be fetched with osc.
30 """
31 return ud.type in ['osc']
32
33 def urldata_init(self, ud, d):
34 if not "module" in ud.parm:
35 raise MissingParameterError('module', ud.url)
36
37 ud.module = ud.parm["module"]
38
39 # Create paths to osc checkouts
40 oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
41 relpath = self._strip_leading_slashes(ud.path)
42 ud.oscdir = oscdir
43 ud.pkgdir = os.path.join(oscdir, ud.host)
44 ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
45
46 if 'rev' in ud.parm:
47 ud.revision = ud.parm['rev']
48 else:
49 pv = d.getVar("PV", False)
50 rev = bb.fetch2.srcrev_internal_helper(ud, d, '')
51 if rev:
52 ud.revision = rev
53 else:
54 ud.revision = ""
55
56 ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision))
57
58 def _buildosccommand(self, ud, d, command):
59 """
60 Build up an ocs commandline based on ud
61 command is "fetch", "update", "info"
62 """
63
64 basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
65
66 proto = ud.parm.get('protocol', 'https')
67
68 options = []
69
70 config = "-c %s" % self.generate_config(ud, d)
71
72 if getattr(ud, 'revision', ''):
73 options.append("-r %s" % ud.revision)
74
75 coroot = self._strip_leading_slashes(ud.path)
76
77 if command == "fetch":
78 osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options))
79 elif command == "update":
80 osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options))
81 elif command == "api_source":
82 osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module)
83 else:
84 raise FetchError("Invalid osc command %s" % command, ud.url)
85
86 return osccmd
87
88 def _latest_revision(self, ud, d, name):
89 """
90 Fetch latest revision for the given package
91 """
92 api_source_cmd = self._buildosccommand(ud, d, "api_source")
93
94 output = runfetchcmd(api_source_cmd, d)
95 match = re.match(r'<directory ?.* rev="(\d+)".*>', output)
96 if match is None:
97 raise FetchError("Unable to parse osc response", ud.url)
98 return match.groups()[0]
99
100 def _revision_key(self, ud, d, name):
101 """
102 Return a unique key for the url
103 """
104 # Collapse adjacent slashes
105 slash_re = re.compile(r"/+")
106 rev = getattr(ud, 'revision', "latest")
107 return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev)
108
109 def download(self, ud, d):
110 """
111 Fetch url
112 """
113
114 logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
115
116 if os.access(ud.moddir, os.R_OK):
117 oscupdatecmd = self._buildosccommand(ud, d, "update")
118 logger.info("Update "+ ud.url)
119 # update sources there
120 logger.debug("Running %s", oscupdatecmd)
121 bb.fetch2.check_network_access(d, oscupdatecmd, ud.url)
122 runfetchcmd(oscupdatecmd, d, workdir=ud.moddir)
123 else:
124 oscfetchcmd = self._buildosccommand(ud, d, "fetch")
125 logger.info("Fetch " + ud.url)
126 # check out sources there
127 bb.utils.mkdirhier(ud.pkgdir)
128 logger.debug("Running %s", oscfetchcmd)
129 bb.fetch2.check_network_access(d, oscfetchcmd, ud.url)
130 runfetchcmd(oscfetchcmd, d, workdir=ud.pkgdir)
131
132 # tar them up to a defined filename
133 runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d,
134 cleanup=[ud.localpath], workdir=os.path.join(ud.pkgdir + ud.path))
135
136 def supports_srcrev(self):
137 return False
138
139 def generate_config(self, ud, d):
140 """
141 Generate a .oscrc to be used for this run.
142 """
143
144 config_path = os.path.join(ud.oscdir, "oscrc")
145 if not os.path.exists(ud.oscdir):
146 bb.utils.mkdirhier(ud.oscdir)
147
148 if (os.path.exists(config_path)):
149 os.remove(config_path)
150
151 f = open(config_path, 'w')
152 proto = ud.parm.get('protocol', 'https')
153 f.write("[general]\n")
154 f.write("apiurl = %s://%s\n" % (proto, ud.host))
155 f.write("su-wrapper = su -c\n")
156 f.write("build-root = %s\n" % d.getVar('WORKDIR'))
157 f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
158 f.write("extra-pkgs = gzip\n")
159 f.write("\n")
160 f.write("[%s://%s]\n" % (proto, ud.host))
161 f.write("user = %s\n" % ud.parm["user"])
162 f.write("pass = %s\n" % ud.parm["pswd"])
163 f.close()
164
165 return config_path
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
deleted file mode 100644
index 3b6fa4b1ec..0000000000
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ /dev/null
@@ -1,267 +0,0 @@
1"""
2BitBake 'Fetch' implementation for perforce
3
4Supported SRC_URI options are:
5
6- module
7 The top-level location to fetch while preserving the remote paths
8
9 The value of module can point to either a directory or a file. The result,
10 in both cases, is that the fetcher will preserve all file paths starting
11 from the module path. That is, the top-level directory in the module value
12 will also be the top-level directory in P4DIR.
13
14- remotepath
15 If the value "keep" is given, the full depot location of each file is
16 preserved in P4DIR. This option overrides the effect of the module option.
17
18"""
19
20# Copyright (C) 2003, 2004 Chris Larson
21# Copyright (C) 2016 Kodak Alaris, Inc.
22#
23# SPDX-License-Identifier: GPL-2.0-only
24#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import os
28import bb
29from bb.fetch2 import FetchMethod
30from bb.fetch2 import FetchError
31from bb.fetch2 import logger
32from bb.fetch2 import runfetchcmd
33
34class PerforceProgressHandler (bb.progress.BasicProgressHandler):
35 """
36 Implements basic progress information for perforce, based on the number of
37 files to be downloaded.
38
39 The p4 print command will print one line per file, therefore it can be used
40 to "count" the number of files already completed and give an indication of
41 the progress.
42 """
43 def __init__(self, d, num_files):
44 self._num_files = num_files
45 self._count = 0
46 super(PerforceProgressHandler, self).__init__(d)
47
48 # Send an initial progress event so the bar gets shown
49 self._fire_progress(-1)
50
51 def write(self, string):
52 self._count = self._count + 1
53
54 percent = int(100.0 * float(self._count) / float(self._num_files))
55
56 # In case something goes wrong, we try to preserve our sanity
57 if percent > 100:
58 percent = 100
59
60 self.update(percent)
61
62 super(PerforceProgressHandler, self).write(string)
63
64class Perforce(FetchMethod):
65 """ Class to fetch from perforce repositories """
66 def supports(self, ud, d):
67 """ Check to see if a given url can be fetched with perforce. """
68 return ud.type in ['p4']
69
70 def urldata_init(self, ud, d):
71 """
72 Initialize perforce specific variables within url data. If P4CONFIG is
73 provided by the env, use it. If P4PORT is specified by the recipe, use
74 its values, which may override the settings in P4CONFIG.
75 """
76 ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
77
78 ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
79
80 path = ud.url.split('://')[1]
81 path = path.split(';')[0]
82 delim = path.find('@');
83 if delim != -1:
84 (ud.user, ud.pswd) = path.split('@')[0].split(':')
85 ud.path = path.split('@')[1]
86 else:
87 ud.path = path
88
89 ud.usingp4config = False
90 p4port = d.getVar('P4PORT')
91
92 if p4port:
93 logger.debug('Using recipe provided P4PORT: %s' % p4port)
94 ud.host = p4port
95 else:
96 logger.debug('Trying to use P4CONFIG to automatically set P4PORT...')
97 ud.usingp4config = True
98 p4cmd = '%s info | grep "Server address"' % ud.basecmd
99 bb.fetch2.check_network_access(d, p4cmd, ud.url)
100 ud.host = runfetchcmd(p4cmd, d, True)
101 ud.host = ud.host.split(': ')[1].strip()
102 logger.debug('Determined P4PORT to be: %s' % ud.host)
103 if not ud.host:
104 raise FetchError('Could not determine P4PORT from P4CONFIG')
105
106 # Fetcher options
107 ud.module = ud.parm.get('module')
108 ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
109
110 if ud.path.find('/...') >= 0:
111 ud.pathisdir = True
112 else:
113 ud.pathisdir = False
114
115 # Avoid using the "/..." syntax in SRC_URI when a module value is given
116 if ud.pathisdir and ud.module:
117 raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
118
119 cleanedpath = ud.path.replace('/...', '').replace('/', '.')
120 cleanedhost = ud.host.replace(':', '.')
121
122 cleanedmodule = ""
123 # Merge the path and module into the final depot location
124 if ud.module:
125 if ud.module.find('/') == 0:
126 raise FetchError('module cannot begin with /')
127 ud.path = os.path.join(ud.path, ud.module)
128
129 # Append the module path to the local pkg name
130 cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
131 cleanedpath += '--%s' % cleanedmodule
132
133 ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
134
135 ud.setup_revisions(d)
136
137 ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
138
139 def _buildp4command(self, ud, d, command, depot_filename=None):
140 """
141 Build a p4 commandline. Valid commands are "changes", "print", and
142 "files". depot_filename is the full path to the file in the depot
143 including the trailing '#rev' value.
144 """
145 p4opt = ""
146
147 if ud.user:
148 p4opt += ' -u "%s"' % (ud.user)
149
150 if ud.pswd:
151 p4opt += ' -P "%s"' % (ud.pswd)
152
153 if ud.host and not ud.usingp4config:
154 p4opt += ' -p %s' % (ud.host)
155
156 if hasattr(ud, 'revision') and ud.revision:
157 pathnrev = '%s@%s' % (ud.path, ud.revision)
158 else:
159 pathnrev = '%s' % (ud.path)
160
161 if depot_filename:
162 if ud.keepremotepath:
163 # preserve everything, remove the leading //
164 filename = depot_filename.lstrip('/')
165 elif ud.module:
166 # remove everything up to the module path
167 modulepath = ud.module.rstrip('/...')
168 filename = depot_filename[depot_filename.rfind(modulepath):]
169 elif ud.pathisdir:
170 # Remove leading (visible) path to obtain the filepath
171 filename = depot_filename[len(ud.path)-1:]
172 else:
173 # Remove everything, except the filename
174 filename = depot_filename[depot_filename.rfind('/'):]
175
176 filename = filename[:filename.find('#')] # Remove trailing '#rev'
177
178 if command == 'changes':
179 p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
180 elif command == 'print':
181 if depot_filename is not None:
182 p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
183 else:
184 raise FetchError('No depot file name provided to p4 %s' % command, ud.url)
185 elif command == 'files':
186 p4cmd = '%s%s files //%s' % (ud.basecmd, p4opt, pathnrev)
187 else:
188 raise FetchError('Invalid p4 command %s' % command, ud.url)
189
190 return p4cmd
191
192 def _p4listfiles(self, ud, d):
193 """
194 Return a list of the file names which are present in the depot using the
195 'p4 files' command, including trailing '#rev' file revision indicator
196 """
197 p4cmd = self._buildp4command(ud, d, 'files')
198 bb.fetch2.check_network_access(d, p4cmd, ud.url)
199 p4fileslist = runfetchcmd(p4cmd, d, True)
200 p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
201
202 if not p4fileslist:
203 raise FetchError('Unable to fetch listing of p4 files from %s@%s' % (ud.host, ud.path))
204
205 count = 0
206 filelist = []
207
208 for filename in p4fileslist:
209 item = filename.split(' - ')
210 lastaction = item[1].split()
211 logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0]))
212 if lastaction[0] == 'delete':
213 continue
214 filelist.append(item[0])
215
216 return filelist
217
218 def download(self, ud, d):
219 """ Get the list of files, fetch each one """
220 filelist = self._p4listfiles(ud, d)
221 if not filelist:
222 raise FetchError('No files found in depot %s@%s' % (ud.host, ud.path))
223
224 bb.utils.remove(ud.pkgdir, True)
225 bb.utils.mkdirhier(ud.pkgdir)
226
227 progresshandler = PerforceProgressHandler(d, len(filelist))
228
229 for afile in filelist:
230 p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
231 bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
232 runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
233
234 runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
235
236 def clean(self, ud, d):
237 """ Cleanup p4 specific files and dirs"""
238 bb.utils.remove(ud.localpath)
239 bb.utils.remove(ud.pkgdir, True)
240
241 def supports_srcrev(self):
242 return True
243
244 def _revision_key(self, ud, d, name):
245 """ Return a unique key for the url """
246 return 'p4:%s' % ud.pkgdir
247
248 def _latest_revision(self, ud, d, name):
249 """ Return the latest upstream scm revision number """
250 p4cmd = self._buildp4command(ud, d, "changes")
251 bb.fetch2.check_network_access(d, p4cmd, ud.url)
252 tip = runfetchcmd(p4cmd, d, True)
253
254 if not tip:
255 raise FetchError('Could not determine the latest perforce changelist')
256
257 tipcset = tip.split(' ')[1]
258 logger.debug('p4 tip found to be changelist %s' % tipcset)
259 return tipcset
260
261 def sortable_revision(self, ud, d, name):
262 """ Return a sortable revision number """
263 return False, self._build_revision(ud, d)
264
265 def _build_revision(self, ud, d):
266 return ud.revision
267
diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
deleted file mode 100644
index fa4cb8149b..0000000000
--- a/bitbake/lib/bb/fetch2/repo.py
+++ /dev/null
@@ -1,87 +0,0 @@
1"""
2BitBake "Fetch" repo (git) implementation
3
4"""
5
6# Copyright (C) 2009 Tom Rini <trini@embeddedalley.com>
7#
8# Based on git.py which is:
9# Copyright (C) 2005 Richard Purdie
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13
14import os
15import bb
16from bb.fetch2 import FetchMethod
17from bb.fetch2 import runfetchcmd
18from bb.fetch2 import logger
19
20class Repo(FetchMethod):
21 """Class to fetch a module or modules from repo (git) repositories"""
22 def supports(self, ud, d):
23 """
24 Check to see if a given url can be fetched with repo.
25 """
26 return ud.type in ["repo"]
27
28 def urldata_init(self, ud, d):
29 """
30 We don"t care about the git rev of the manifests repository, but
31 we do care about the manifest to use. The default is "default".
32 We also care about the branch or tag to be used. The default is
33 "master".
34 """
35
36 ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo"
37
38 ud.proto = ud.parm.get('protocol', 'git')
39 ud.branch = ud.parm.get('branch', 'master')
40 ud.manifest = ud.parm.get('manifest', 'default.xml')
41 if not ud.manifest.endswith('.xml'):
42 ud.manifest += '.xml'
43
44 ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
45
46 def download(self, ud, d):
47 """Fetch url"""
48
49 if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
50 logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
51 return
52
53 repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
54 gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
55 codir = os.path.join(repodir, gitsrcname, ud.manifest)
56
57 if ud.user:
58 username = ud.user + "@"
59 else:
60 username = ""
61
62 repodir = os.path.join(codir, "repo")
63 bb.utils.mkdirhier(repodir)
64 if not os.path.exists(os.path.join(repodir, ".repo")):
65 bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
66 runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
67
68 bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
69 runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)
70
71 scmdata = ud.parm.get("scmdata", "")
72 if scmdata == "keep":
73 tar_flags = ""
74 else:
75 tar_flags = "--exclude='.repo' --exclude='.git'"
76
77 # Create a cache
78 runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir)
79
80 def supports_srcrev(self):
81 return False
82
83 def _build_revision(self, ud, d):
84 return ud.manifest
85
86 def _want_sortable_revision(self, ud, d):
87 return False
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py
deleted file mode 100644
index 22c0538139..0000000000
--- a/bitbake/lib/bb/fetch2/s3.py
+++ /dev/null
@@ -1,124 +0,0 @@
1"""
2BitBake 'Fetch' implementation for Amazon AWS S3.
3
4Class for fetching files from Amazon S3 using the AWS Command Line Interface.
5The aws tool must be correctly installed and configured prior to use.
6
7"""
8
9# Copyright (C) 2017, Andre McCurdy <armccurdy@gmail.com>
10#
11# Based in part on bb.fetch2.wget:
12# Copyright (C) 2003, 2004 Chris Larson
13#
14# SPDX-License-Identifier: GPL-2.0-only
15#
16# Based on functions from the base bb module, Copyright 2003 Holger Schurig
17
18import os
19import bb
20import urllib.request, urllib.parse, urllib.error
21import re
22from bb.fetch2 import FetchMethod
23from bb.fetch2 import FetchError
24from bb.fetch2 import runfetchcmd
25
26def convertToBytes(value, unit):
27 value = float(value)
28 if (unit == "KiB"):
29 value = value*1024.0;
30 elif (unit == "MiB"):
31 value = value*1024.0*1024.0;
32 elif (unit == "GiB"):
33 value = value*1024.0*1024.0*1024.0;
34 return value
35
36class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
37 """
38 Extract progress information from s3 cp output, e.g.:
39 Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
40 """
41 def __init__(self, d):
42 super(S3ProgressHandler, self).__init__(d)
43 # Send an initial progress event so the bar gets shown
44 self._fire_progress(0)
45
46 def writeline(self, line):
47 percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
48 if percs:
49 completed = (percs[-1][0])
50 completedUnit = (percs[-1][1])
51 total = (percs[-1][2])
52 totalUnit = (percs[-1][3])
53 completed = convertToBytes(completed, completedUnit)
54 total = convertToBytes(total, totalUnit)
55 progress = (completed/total)*100.0
56 rate = percs[-1][4]
57 self.update(progress, rate)
58 return False
59 return True
60
61
62class S3(FetchMethod):
63 """Class to fetch urls via 'aws s3'"""
64
65 def supports(self, ud, d):
66 """
67 Check to see if a given url can be fetched with s3.
68 """
69 return ud.type in ['s3']
70
71 def recommends_checksum(self, urldata):
72 return True
73
74 def urldata_init(self, ud, d):
75 if 'downloadfilename' in ud.parm:
76 ud.basename = ud.parm['downloadfilename']
77 else:
78 ud.basename = os.path.basename(ud.path)
79
80 ud.localfile = ud.basename
81
82 ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
83
84 def download(self, ud, d):
85 """
86 Fetch urls
87 Assumes localpath was called first
88 """
89
90 cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
91 bb.fetch2.check_network_access(d, cmd, ud.url)
92
93 progresshandler = S3ProgressHandler(d)
94 runfetchcmd(cmd, d, False, log=progresshandler)
95
96 # Additional sanity checks copied from the wget class (although there
97 # are no known issues which mean these are required, treat the aws cli
98 # tool with a little healthy suspicion).
99
100 if not os.path.exists(ud.localpath):
101 raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
102
103 if os.path.getsize(ud.localpath) == 0:
104 os.remove(ud.localpath)
105 raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
106
107 return True
108
109 def checkstatus(self, fetch, ud, d):
110 """
111 Check the status of a URL
112 """
113
114 cmd = '%s ls s3://%s%s' % (ud.basecmd, ud.host, ud.path)
115 bb.fetch2.check_network_access(d, cmd, ud.url)
116 output = runfetchcmd(cmd, d)
117
118 # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
119 # is not found, so check output of the command to confirm success.
120
121 if not output:
122 raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
123
124 return True
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
deleted file mode 100644
index bee71a0d0d..0000000000
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ /dev/null
@@ -1,112 +0,0 @@
1"""
2BitBake SFTP Fetch implementation
3
4Class for fetching files via SFTP. It tries to adhere to the (now
5expired) IETF Internet Draft for "Uniform Resource Identifier (URI)
6Scheme for Secure File Transfer Protocol (SFTP) and Secure Shell
7(SSH)" (SECSH URI).
8
9It uses SFTP (as to adhere to the SECSH URI specification). It only
10supports key based authentication, not password. This class, unlike
11the SSH fetcher, does not support fetching a directory tree from the
12remote.
13
14 http://tools.ietf.org/html/draft-ietf-secsh-scp-sftp-ssh-uri-04
15 https://www.iana.org/assignments/uri-schemes/prov/sftp
16 https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13
17
18Please note that '/' is used as host path seperator, and not ":"
19as you may be used to from the scp/sftp commands. You can use a
20~ (tilde) to specify a path relative to your home directory.
21(The /~user/ syntax, for specyfing a path relative to another
22user's home directory is not supported.) Note that the tilde must
23still follow the host path seperator ("/"). See exampels below.
24
25Example SRC_URIs:
26
27SRC_URI = "sftp://host.example.com/dir/path.file.txt"
28
29A path relative to your home directory.
30
31SRC_URI = "sftp://host.example.com/~/dir/path.file.txt"
32
33You can also specify a username (specyfing password in the
34URI is not supported, use SSH keys to authenticate):
35
36SRC_URI = "sftp://user@host.example.com/dir/path.file.txt"
37
38"""
39
40# Copyright (C) 2013, Olof Johansson <olof.johansson@axis.com>
41#
42# Based in part on bb.fetch2.wget:
43# Copyright (C) 2003, 2004 Chris Larson
44#
45# SPDX-License-Identifier: GPL-2.0-only
46#
47# Based on functions from the base bb module, Copyright 2003 Holger Schurig
48
49import os
50import bb
51import urllib.request, urllib.parse, urllib.error
52from bb.fetch2 import URI
53from bb.fetch2 import FetchMethod
54from bb.fetch2 import runfetchcmd
55
56class SFTP(FetchMethod):
57 """Class to fetch urls via 'sftp'"""
58
59 def supports(self, ud, d):
60 """
61 Check to see if a given url can be fetched with sftp.
62 """
63 return ud.type in ['sftp']
64
65 def recommends_checksum(self, urldata):
66 return True
67
68 def urldata_init(self, ud, d):
69 if 'protocol' in ud.parm and ud.parm['protocol'] == 'git':
70 raise bb.fetch2.ParameterError(
71 "Invalid protocol - if you wish to fetch from a " +
72 "git repository using ssh, you need to use the " +
73 "git:// prefix with protocol=ssh", ud.url)
74
75 if 'downloadfilename' in ud.parm:
76 ud.basename = ud.parm['downloadfilename']
77 else:
78 ud.basename = os.path.basename(ud.path)
79
80 ud.localfile = ud.basename
81
82 def download(self, ud, d):
83 """Fetch urls"""
84
85 urlo = URI(ud.url)
86 basecmd = 'sftp -oBatchMode=yes'
87 port = ''
88 if urlo.port:
89 port = '-P %d' % urlo.port
90 urlo.port = None
91
92 dldir = d.getVar('DL_DIR')
93 lpath = os.path.join(dldir, ud.localfile)
94
95 user = ''
96 if urlo.userinfo:
97 user = urlo.userinfo + '@'
98
99 path = urlo.path
100
101 # Supoprt URIs relative to the user's home directory, with
102 # the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>).
103 if path[:3] == '/~/':
104 path = path[3:]
105
106 remote = '"%s%s:%s"' % (user, urlo.hostname, path)
107
108 cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
109
110 bb.fetch2.check_network_access(d, cmd, ud.url)
111 runfetchcmd(cmd, d)
112 return True
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
deleted file mode 100644
index 2a0f2cb44b..0000000000
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ /dev/null
@@ -1,154 +0,0 @@
1'''
2BitBake 'Fetch' implementations
3
4This implementation is for Secure Shell (SSH), and attempts to comply with the
5IETF secsh internet draft:
6 http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/
7
8 Currently does not support the sftp parameters, as this uses scp
9 Also does not support the 'fingerprint' connection parameter.
10
11 Please note that '/' is used as host, path separator not ':' as you may
12 be used to, also '~' can be used to specify user HOME, but again after '/'
13
14 Example SRC_URI:
15 SRC_URI = "ssh://user@host.example.com/dir/path/file.txt"
16 SRC_URI = "ssh://user@host.example.com/~/file.txt"
17'''
18
19# Copyright (C) 2006 OpenedHand Ltd.
20#
21#
22# Based in part on svk.py:
23# Copyright (C) 2006 Holger Hans Peter Freyther
24# Based on svn.py:
25# Copyright (C) 2003, 2004 Chris Larson
26# Based on functions from the base bb module:
27# Copyright 2003 Holger Schurig
28#
29#
30# SPDX-License-Identifier: GPL-2.0-only
31#
32
33import re, os
34from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
35import urllib
36
37
38__pattern__ = re.compile(r'''
39 \s* # Skip leading whitespace
40 ssh:// # scheme
41 ( # Optional username/password block
42 (?P<user>\S+) # username
43 (:(?P<pass>\S+))? # colon followed by the password (optional)
44 (?P<cparam>(;[^;]+)*)? # connection parameters block (optional)
45 @
46 )?
47 (?P<host>\S+?) # non-greedy match of the host
48 (:(?P<port>[0-9]+))? # colon followed by the port (optional)
49 /
50 (?P<path>[^;]+) # path on the remote system, may be absolute or relative,
51 # and may include the use of '~' to reference the remote home
52 # directory
53 (?P<sparam>(;[^;]+)*)? # parameters block (optional)
54 $
55''', re.VERBOSE)
56
57class SSH(FetchMethod):
58 '''Class to fetch a module or modules via Secure Shell'''
59
60 def supports(self, urldata, d):
61 return __pattern__.match(urldata.url) is not None
62
63 def supports_checksum(self, urldata):
64 return False
65
66 def urldata_init(self, urldata, d):
67 if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git':
68 raise ParameterError(
69 "Invalid protocol - if you wish to fetch from a git " +
70 "repository using ssh, you need to use " +
71 "git:// prefix with protocol=ssh", urldata.url)
72 m = __pattern__.match(urldata.url)
73 path = m.group('path')
74 path = urllib.parse.unquote(path)
75 host = m.group('host')
76 urldata.localfile = os.path.basename(os.path.normpath(path))
77
78 def download(self, urldata, d):
79 dldir = d.getVar('DL_DIR')
80
81 m = __pattern__.match(urldata.url)
82 path = m.group('path')
83 host = m.group('host')
84 port = m.group('port')
85 user = m.group('user')
86 password = m.group('pass')
87
88 if port:
89 portarg = '-P %s' % port
90 else:
91 portarg = ''
92
93 if user:
94 fr = user
95 if password:
96 fr += ':%s' % password
97 fr += '@%s' % host
98 else:
99 fr = host
100
101 if path[0] != '~':
102 path = '/%s' % path
103 path = urllib.parse.unquote(path)
104
105 fr += ':%s' % path
106
107 cmd = 'scp -B -r %s %s %s/' % (
108 portarg,
109 fr,
110 dldir
111 )
112
113 check_network_access(d, cmd, urldata.url)
114
115 runfetchcmd(cmd, d)
116
117 def checkstatus(self, fetch, urldata, d):
118 """
119 Check the status of the url
120 """
121 m = __pattern__.match(urldata.url)
122 path = m.group('path')
123 host = m.group('host')
124 port = m.group('port')
125 user = m.group('user')
126 password = m.group('pass')
127
128 if port:
129 portarg = '-P %s' % port
130 else:
131 portarg = ''
132
133 if user:
134 fr = user
135 if password:
136 fr += ':%s' % password
137 fr += '@%s' % host
138 else:
139 fr = host
140
141 if path[0] != '~':
142 path = '/%s' % path
143 path = urllib.parse.unquote(path)
144
145 cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % (
146 portarg,
147 fr,
148 path
149 )
150
151 check_network_access(d, cmd, urldata.url)
152 runfetchcmd(cmd, d)
153
154 return True
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
deleted file mode 100644
index 0852108e7d..0000000000
--- a/bitbake/lib/bb/fetch2/svn.py
+++ /dev/null
@@ -1,215 +0,0 @@
1"""
2BitBake 'Fetch' implementation for svn.
3
4"""
5
6# Copyright (C) 2003, 2004 Chris Larson
7# Copyright (C) 2004 Marcin Juszkiewicz
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11# Based on functions from the base bb module, Copyright 2003 Holger Schurig
12
13import os
14import bb
15import re
16from bb.fetch2 import FetchMethod
17from bb.fetch2 import FetchError
18from bb.fetch2 import MissingParameterError
19from bb.fetch2 import runfetchcmd
20from bb.fetch2 import logger
21
22class Svn(FetchMethod):
23 """Class to fetch a module or modules from svn repositories"""
24 def supports(self, ud, d):
25 """
26 Check to see if a given url can be fetched with svn.
27 """
28 return ud.type in ['svn']
29
30 def urldata_init(self, ud, d):
31 """
32 init svn specific variable within url data
33 """
34 if not "module" in ud.parm:
35 raise MissingParameterError('module', ud.url)
36
37 ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert"
38
39 ud.module = ud.parm["module"]
40
41 if not "path_spec" in ud.parm:
42 ud.path_spec = ud.module
43 else:
44 ud.path_spec = ud.parm["path_spec"]
45
46 # Create paths to svn checkouts
47 svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
48 relpath = self._strip_leading_slashes(ud.path)
49 ud.pkgdir = os.path.join(svndir, ud.host, relpath)
50 ud.moddir = os.path.join(ud.pkgdir, ud.path_spec)
51 # Protects the repository from concurrent updates, e.g. from two
52 # recipes fetching different revisions at the same time
53 ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
54
55 ud.setup_revisions(d)
56
57 if 'rev' in ud.parm:
58 ud.revision = ud.parm['rev']
59
60 # Whether to use the @REV peg-revision syntax in the svn command or not
61 ud.pegrevision = True
62 if 'nopegrevision' in ud.parm:
63 ud.pegrevision = False
64
65 ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision]))
66
67 def _buildsvncommand(self, ud, d, command):
68 """
69 Build up an svn commandline based on ud
70 command is "fetch", "update", "info"
71 """
72
73 proto = ud.parm.get('protocol', 'svn')
74
75 svn_ssh = None
76 if proto == "svn+ssh" and "ssh" in ud.parm:
77 svn_ssh = ud.parm["ssh"]
78
79 svnroot = ud.host + ud.path
80
81 options = []
82
83 options.append("--no-auth-cache")
84
85 if ud.user:
86 options.append("--username %s" % ud.user)
87
88 if ud.pswd:
89 options.append("--password %s" % ud.pswd)
90
91 if command == "info":
92 svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
93 elif command == "log1":
94 svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
95 else:
96 suffix = ""
97
98 # externals may be either 'allowed' or 'nowarn', but not both. Allowed
99 # will not issue a warning, but will log to the debug buffer what has likely
100 # been downloaded by SVN.
101 if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"):
102 options.append("--ignore-externals")
103
104 if ud.revision:
105 options.append("-r %s" % ud.revision)
106 if ud.pegrevision:
107 suffix = "@%s" % (ud.revision)
108
109 if command == "fetch":
110 transportuser = ud.parm.get("transportuser", "")
111 svncmd = "%s co %s %s://%s%s/%s%s %s" % (ud.basecmd, " ".join(options), proto, transportuser, svnroot, ud.module, suffix, ud.path_spec)
112 elif command == "update":
113 svncmd = "%s update %s" % (ud.basecmd, " ".join(options))
114 else:
115 raise FetchError("Invalid svn command %s" % command, ud.url)
116
117 if svn_ssh:
118 svncmd = "SVN_SSH=\"%s\" %s" % (svn_ssh, svncmd)
119
120 return svncmd
121
122 def download(self, ud, d):
123 """Fetch url"""
124
125 logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
126
127 lf = bb.utils.lockfile(ud.svnlock)
128
129 try:
130 if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
131 svncmd = self._buildsvncommand(ud, d, "update")
132 logger.info("Update " + ud.url)
133 # We need to attempt to run svn upgrade first in case its an older working format
134 try:
135 runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
136 except FetchError:
137 pass
138 logger.debug("Running %s", svncmd)
139 bb.fetch2.check_network_access(d, svncmd, ud.url)
140 runfetchcmd(svncmd, d, workdir=ud.moddir)
141 else:
142 svncmd = self._buildsvncommand(ud, d, "fetch")
143 logger.info("Fetch " + ud.url)
144 # check out sources there
145 bb.utils.mkdirhier(ud.pkgdir)
146 logger.debug("Running %s", svncmd)
147 bb.fetch2.check_network_access(d, svncmd, ud.url)
148 runfetchcmd(svncmd, d, workdir=ud.pkgdir)
149
150 if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
151 # Warn the user if this had externals (won't catch them all)
152 output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir)
153 if output:
154 if "--ignore-externals" in svncmd.split():
155 bb.warn("%s contains svn:externals." % ud.url)
156 bb.warn("These should be added to the recipe SRC_URI as necessary.")
157 bb.warn("svn fetch has ignored externals:\n%s" % output)
158 bb.warn("To disable this warning add ';externals=nowarn' to the url.")
159 else:
160 bb.debug(1, "svn repository has externals:\n%s" % output)
161
162 scmdata = ud.parm.get("scmdata", "")
163 if scmdata == "keep":
164 tar_flags = ""
165 else:
166 tar_flags = "--exclude='.svn'"
167
168 # tar them up to a defined filename
169 runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
170 cleanup=[ud.localpath], workdir=ud.pkgdir)
171 finally:
172 bb.utils.unlockfile(lf)
173
174 def clean(self, ud, d):
175 """ Clean SVN specific files and dirs """
176
177 bb.utils.remove(ud.localpath)
178 bb.utils.remove(ud.moddir, True)
179
180
181 def supports_srcrev(self):
182 return True
183
184 def _revision_key(self, ud, d, name):
185 """
186 Return a unique key for the url
187 """
188 return "svn:" + ud.moddir
189
190 def _latest_revision(self, ud, d, name):
191 """
192 Return the latest upstream revision number
193 """
194 bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"), ud.url)
195
196 output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True)
197
198 # skip the first line, as per output of svn log
199 # then we expect the revision on the 2nd line
200 revision = re.search('^r([0-9]*)', output.splitlines()[1]).group(1)
201
202 return revision
203
204 def sortable_revision(self, ud, d, name):
205 """
206 Return a sortable revision number which in our case is the revision number
207 """
208
209 return False, self._build_revision(ud, d)
210
211 def _build_revision(self, ud, d):
212 return ud.revision
213
214 def supports_checksum(self, urldata):
215 return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
deleted file mode 100644
index 4d19e2134b..0000000000
--- a/bitbake/lib/bb/fetch2/wget.py
+++ /dev/null
@@ -1,693 +0,0 @@
1"""
2BitBake 'Fetch' implementations
3
4Classes for obtaining upstream sources for the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import shlex
16import re
17import tempfile
18import os
19import errno
20import bb
21import bb.progress
22import socket
23import http.client
24import urllib.request, urllib.parse, urllib.error
25from bb.fetch2 import FetchMethod
26from bb.fetch2 import FetchError
27from bb.fetch2 import logger
28from bb.fetch2 import runfetchcmd
29from bs4 import BeautifulSoup
30from bs4 import SoupStrainer
31
32class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
33 """
34 Extract progress information from wget output.
35 Note: relies on --progress=dot (with -v or without -q/-nv) being
36 specified on the wget command line.
37 """
38 def __init__(self, d):
39 super(WgetProgressHandler, self).__init__(d)
40 # Send an initial progress event so the bar gets shown
41 self._fire_progress(0)
42
43 def writeline(self, line):
44 percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
45 if percs:
46 progress = int(percs[-1][0])
47 rate = percs[-1][1] + '/s'
48 self.update(progress, rate)
49 return False
50 return True
51
52
53class Wget(FetchMethod):
54 """Class to fetch urls via 'wget'"""
55
56 def check_certs(self, d):
57 """
58 Should certificates be checked?
59 """
60 return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
61
62 def supports(self, ud, d):
63 """
64 Check to see if a given url can be fetched with wget.
65 """
66 return ud.type in ['http', 'https', 'ftp', 'ftps']
67
68 def recommends_checksum(self, urldata):
69 return True
70
71 def urldata_init(self, ud, d):
72 if 'protocol' in ud.parm:
73 if ud.parm['protocol'] == 'git':
74 raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url)
75
76 if 'downloadfilename' in ud.parm:
77 ud.basename = ud.parm['downloadfilename']
78 else:
79 ud.basename = os.path.basename(ud.path)
80
81 ud.localfile = ud.basename
82 if not ud.localfile:
83 ud.localfile = ud.host + ud.path.replace("/", ".")
84
85 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100"
86
87 if ud.type == 'ftp' or ud.type == 'ftps':
88 self.basecmd += " --passive-ftp"
89
90 if not self.check_certs(d):
91 self.basecmd += " --no-check-certificate"
92
93 def _runwget(self, ud, d, command, quiet, workdir=None):
94
95 progresshandler = WgetProgressHandler(d)
96
97 logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
98 bb.fetch2.check_network_access(d, command, ud.url)
99 runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir)
100
101 def download(self, ud, d):
102 """Fetch urls"""
103
104 fetchcmd = self.basecmd
105
106 dldir = os.path.realpath(d.getVar("DL_DIR"))
107 localpath = os.path.join(dldir, ud.localfile) + ".tmp"
108 bb.utils.mkdirhier(os.path.dirname(localpath))
109 fetchcmd += " --output-document=%s" % shlex.quote(localpath)
110
111 if ud.user and ud.pswd:
112 fetchcmd += " --auth-no-challenge"
113 if ud.parm.get("redirectauth", "1") == "1":
114 # An undocumented feature of wget is that if the
115 # username/password are specified on the URI, wget will only
116 # send the Authorization header to the first host and not to
117 # any hosts that it is redirected to. With the increasing
118 # usage of temporary AWS URLs, this difference now matters as
119 # AWS will reject any request that has authentication both in
120 # the query parameters (from the redirect) and in the
121 # Authorization header.
122 fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
123
124 uri = ud.url.split(";")[0]
125 fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
126 self._runwget(ud, d, fetchcmd, False)
127
128 # Sanity check since wget can pretend it succeed when it didn't
129 # Also, this used to happen if sourceforge sent us to the mirror page
130 if not os.path.exists(localpath):
131 raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
132
133 if os.path.getsize(localpath) == 0:
134 os.remove(localpath)
135 raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
136
137 # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
138 # original file, which might be a race (imagine two recipes referencing the same
139 # source, one with an incorrect checksum)
140 bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
141
142 # Remove the ".tmp" and move the file into position atomically
143 # Our lock prevents multiple writers but mirroring code may grab incomplete files
144 os.rename(localpath, localpath[:-4])
145
146 return True
147
148 def checkstatus(self, fetch, ud, d, try_again=True):
149 class HTTPConnectionCache(http.client.HTTPConnection):
150 if fetch.connection_cache:
151 def connect(self):
152 """Connect to the host and port specified in __init__."""
153
154 sock = fetch.connection_cache.get_connection(self.host, self.port)
155 if sock:
156 self.sock = sock
157 else:
158 self.sock = socket.create_connection((self.host, self.port),
159 self.timeout, self.source_address)
160 fetch.connection_cache.add_connection(self.host, self.port, self.sock)
161
162 if self._tunnel_host:
163 self._tunnel()
164
165 class CacheHTTPHandler(urllib.request.HTTPHandler):
166 def http_open(self, req):
167 return self.do_open(HTTPConnectionCache, req)
168
169 def do_open(self, http_class, req):
170 """Return an addinfourl object for the request, using http_class.
171
172 http_class must implement the HTTPConnection API from httplib.
173 The addinfourl return value is a file-like object. It also
174 has methods and attributes including:
175 - info(): return a mimetools.Message object for the headers
176 - geturl(): return the original request URL
177 - code: HTTP status code
178 """
179 host = req.host
180 if not host:
181 raise urllib.error.URLError('no host given')
182
183 h = http_class(host, timeout=req.timeout) # will parse host:port
184 h.set_debuglevel(self._debuglevel)
185
186 headers = dict(req.unredirected_hdrs)
187 headers.update(dict((k, v) for k, v in list(req.headers.items())
188 if k not in headers))
189
190 # We want to make an HTTP/1.1 request, but the addinfourl
191 # class isn't prepared to deal with a persistent connection.
192 # It will try to read all remaining data from the socket,
193 # which will block while the server waits for the next request.
194 # So make sure the connection gets closed after the (only)
195 # request.
196
197 # Don't close connection when connection_cache is enabled,
198 if fetch.connection_cache is None:
199 headers["Connection"] = "close"
200 else:
201 headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
202
203 headers = dict(
204 (name.title(), val) for name, val in list(headers.items()))
205
206 if req._tunnel_host:
207 tunnel_headers = {}
208 proxy_auth_hdr = "Proxy-Authorization"
209 if proxy_auth_hdr in headers:
210 tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
211 # Proxy-Authorization should not be sent to origin
212 # server.
213 del headers[proxy_auth_hdr]
214 h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
215
216 try:
217 h.request(req.get_method(), req.selector, req.data, headers)
218 except socket.error as err: # XXX what error?
219 # Don't close connection when cache is enabled.
220 # Instead, try to detect connections that are no longer
221 # usable (for example, closed unexpectedly) and remove
222 # them from the cache.
223 if fetch.connection_cache is None:
224 h.close()
225 elif isinstance(err, OSError) and err.errno == errno.EBADF:
226 # This happens when the server closes the connection despite the Keep-Alive.
227 # Apparently urllib then uses the file descriptor, expecting it to be
228 # connected, when in reality the connection is already gone.
229 # We let the request fail and expect it to be
230 # tried once more ("try_again" in check_status()),
231 # with the dead connection removed from the cache.
232 # If it still fails, we give up, which can happen for bad
233 # HTTP proxy settings.
234 fetch.connection_cache.remove_connection(h.host, h.port)
235 raise urllib.error.URLError(err)
236 else:
237 try:
238 r = h.getresponse()
239 except TimeoutError as e:
240 if fetch.connection_cache:
241 fetch.connection_cache.remove_connection(h.host, h.port)
242 raise TimeoutError(e)
243
244 # Pick apart the HTTPResponse object to get the addinfourl
245 # object initialized properly.
246
247 # Wrap the HTTPResponse object in socket's file object adapter
248 # for Windows. That adapter calls recv(), so delegate recv()
249 # to read(). This weird wrapping allows the returned object to
250 # have readline() and readlines() methods.
251
252 # XXX It might be better to extract the read buffering code
253 # out of socket._fileobject() and into a base class.
254 r.recv = r.read
255
256 # no data, just have to read
257 r.read()
258 class fp_dummy(object):
259 def read(self):
260 return ""
261 def readline(self):
262 return ""
263 def close(self):
264 pass
265 closed = False
266
267 resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url())
268 resp.code = r.status
269 resp.msg = r.reason
270
271 # Close connection when server request it.
272 if fetch.connection_cache is not None:
273 if 'Connection' in r.msg and r.msg['Connection'] == 'close':
274 fetch.connection_cache.remove_connection(h.host, h.port)
275
276 return resp
277
278 class HTTPMethodFallback(urllib.request.BaseHandler):
279 """
280 Fallback to GET if HEAD is not allowed (405 HTTP error)
281 """
282 def http_error_405(self, req, fp, code, msg, headers):
283 fp.read()
284 fp.close()
285
286 if req.get_method() != 'GET':
287 newheaders = dict((k, v) for k, v in list(req.headers.items())
288 if k.lower() not in ("content-length", "content-type"))
289 return self.parent.open(urllib.request.Request(req.get_full_url(),
290 headers=newheaders,
291 origin_req_host=req.origin_req_host,
292 unverifiable=True))
293
294 raise urllib.request.HTTPError(req, code, msg, headers, None)
295
296 # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
297 # Forbidden when they actually mean 405 Method Not Allowed.
298 http_error_403 = http_error_405
299
300
301 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
302 """
303 urllib2.HTTPRedirectHandler before 3.13 has two flaws:
304
305 It resets the method to GET on redirect when we want to follow
306 redirects using the original method (typically HEAD). This was fixed
307 in 759e8e7.
308
309 It also doesn't handle 308 (Permanent Redirect). This was fixed in
310 c379bc5.
311
312 Until we depend on Python 3.13 onwards, copy the redirect_request
313 method to fix these issues.
314 """
315 def redirect_request(self, req, fp, code, msg, headers, newurl):
316 m = req.get_method()
317 if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
318 or code in (301, 302, 303) and m == "POST")):
319 raise urllib.HTTPError(req.full_url, code, msg, headers, fp)
320
321 # Strictly (according to RFC 2616), 301 or 302 in response to
322 # a POST MUST NOT cause a redirection without confirmation
323 # from the user (of urllib.request, in this case). In practice,
324 # essentially all clients do redirect in this case, so we do
325 # the same.
326
327 # Be conciliant with URIs containing a space. This is mainly
328 # redundant with the more complete encoding done in http_error_302(),
329 # but it is kept for compatibility with other callers.
330 newurl = newurl.replace(' ', '%20')
331
332 CONTENT_HEADERS = ("content-length", "content-type")
333 newheaders = {k: v for k, v in req.headers.items()
334 if k.lower() not in CONTENT_HEADERS}
335 return urllib.request.Request(newurl,
336 method="HEAD" if m == "HEAD" else "GET",
337 headers=newheaders,
338 origin_req_host=req.origin_req_host,
339 unverifiable=True)
340
341 http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
342
343 # We need to update the environment here as both the proxy and HTTPS
344 # handlers need variables set. The proxy needs http_proxy and friends to
345 # be set, and HTTPSHandler ends up calling into openssl to load the
346 # certificates. In buildtools configurations this will be looking at the
347 # wrong place for certificates by default: we set SSL_CERT_FILE to the
348 # right location in the buildtools environment script but as BitBake
349 # prunes prunes the environment this is lost. When binaries are executed
350 # runfetchcmd ensures these values are in the environment, but this is
351 # pure Python so we need to update the environment.
352 #
353 # Avoid tramping the environment too much by using bb.utils.environment
354 # to scope the changes to the build_opener request, which is when the
355 # environment lookups happen.
356 newenv = bb.fetch2.get_fetcher_environment(d)
357
358 with bb.utils.environment(**newenv):
359 import ssl
360
361 if self.check_certs(d):
362 context = ssl.create_default_context()
363 else:
364 context = ssl._create_unverified_context()
365
366 handlers = [FixedHTTPRedirectHandler,
367 HTTPMethodFallback,
368 urllib.request.ProxyHandler(),
369 CacheHTTPHandler(),
370 urllib.request.HTTPSHandler(context=context)]
371 opener = urllib.request.build_opener(*handlers)
372
373 try:
374 parts = urllib.parse.urlparse(ud.url.split(";")[0])
375 if parts.query:
376 uri = "{}://{}{}?{}".format(parts.scheme, parts.netloc, parts.path, parts.query)
377 else:
378 uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path)
379 r = urllib.request.Request(uri)
380 r.get_method = lambda: "HEAD"
381 # Some servers (FusionForge, as used on Alioth) require that the
382 # optional Accept header is set.
383 r.add_header("Accept", "*/*")
384 r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
385 def add_basic_auth(login_str, request):
386 '''Adds Basic auth to http request, pass in login:password as string'''
387 import base64
388 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
389 authheader = "Basic %s" % encodeuser
390 r.add_header("Authorization", authheader)
391
392 if ud.user and ud.pswd:
393 add_basic_auth(ud.user + ':' + ud.pswd, r)
394
395 try:
396 import netrc
397 auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
398 if auth_data:
399 login, _, password = auth_data
400 add_basic_auth("%s:%s" % (login, password), r)
401 except (FileNotFoundError, netrc.NetrcParseError):
402 pass
403
404 with opener.open(r, timeout=100) as response:
405 pass
406 except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
407 if try_again:
408 logger.debug2("checkstatus: trying again")
409 return self.checkstatus(fetch, ud, d, False)
410 else:
411 # debug for now to avoid spamming the logs in e.g. remote sstate searches
412 logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
413 return False
414
415 return True
416
417 def _parse_path(self, regex, s):
418 """
419 Find and group name, version and archive type in the given string s
420 """
421
422 m = regex.search(s)
423 if m:
424 pname = ''
425 pver = ''
426 ptype = ''
427
428 mdict = m.groupdict()
429 if 'name' in mdict.keys():
430 pname = mdict['name']
431 if 'pver' in mdict.keys():
432 pver = mdict['pver']
433 if 'type' in mdict.keys():
434 ptype = mdict['type']
435
436 bb.debug(3, "_parse_path: %s, %s, %s" % (pname, pver, ptype))
437
438 return (pname, pver, ptype)
439
440 return None
441
442 def _modelate_version(self, version):
443 if version[0] in ['.', '-']:
444 if version[1].isdigit():
445 version = version[1] + version[0] + version[2:len(version)]
446 else:
447 version = version[1:len(version)]
448
449 version = re.sub('-', '.', version)
450 version = re.sub('_', '.', version)
451 version = re.sub('(rc)+', '.1000.', version)
452 version = re.sub('(beta)+', '.100.', version)
453 version = re.sub('(alpha)+', '.10.', version)
454 if version[0] == 'v':
455 version = version[1:len(version)]
456 return version
457
458 def _vercmp(self, old, new):
459 """
460 Check whether 'new' is newer than 'old' version. We use existing vercmp() for the
461 purpose. PE is cleared in comparison as it's not for build, and PR is cleared too
462 for simplicity as it's somehow difficult to get from various upstream format
463 """
464
465 (oldpn, oldpv, oldsuffix) = old
466 (newpn, newpv, newsuffix) = new
467
468 # Check for a new suffix type that we have never heard of before
469 if newsuffix:
470 m = self.suffix_regex_comp.search(newsuffix)
471 if not m:
472 bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix))
473 return False
474
475 # Not our package so ignore it
476 if oldpn != newpn:
477 return False
478
479 oldpv = self._modelate_version(oldpv)
480 newpv = self._modelate_version(newpv)
481
482 return bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, ""))
483
484 def _fetch_index(self, uri, ud, d):
485 """
486 Run fetch checkstatus to get directory information
487 """
488 f = tempfile.NamedTemporaryFile()
489 with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
490 fetchcmd = self.basecmd
491 fetchcmd += " --output-document=%s '%s'" % (f.name, uri)
492 try:
493 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
494 fetchresult = f.read()
495 except bb.fetch2.BBFetchException:
496 fetchresult = ""
497
498 return fetchresult
499
500 def _check_latest_version(self, url, package, package_regex, current_version, ud, d):
501 """
502 Return the latest version of a package inside a given directory path
503 If error or no version, return ""
504 """
505 valid = 0
506 version = ['', '', '']
507
508 bb.debug(3, "VersionURL: %s" % (url))
509 soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a"))
510 if not soup:
511 bb.debug(3, "*** %s NO SOUP" % (url))
512 return ""
513
514 for line in soup.find_all('a', href=True):
515 bb.debug(3, "line['href'] = '%s'" % (line['href']))
516 bb.debug(3, "line = '%s'" % (str(line)))
517
518 newver = self._parse_path(package_regex, line['href'])
519 if not newver:
520 newver = self._parse_path(package_regex, str(line))
521
522 if newver:
523 bb.debug(3, "Upstream version found: %s" % newver[1])
524 if valid == 0:
525 version = newver
526 valid = 1
527 elif self._vercmp(version, newver) < 0:
528 version = newver
529
530 pupver = re.sub('_', '.', version[1])
531
532 bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" %
533 (package, pupver or "N/A", current_version[1]))
534
535 if valid:
536 return pupver
537
538 return ""
539
540 def _check_latest_version_by_dir(self, dirver, package, package_regex, current_version, ud, d):
541 """
542 Scan every directory in order to get upstream version.
543 """
544 version_dir = ['', '', '']
545 version = ['', '', '']
546
547 dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
548 s = dirver_regex.search(dirver)
549 if s:
550 version_dir[1] = s.group('ver')
551 else:
552 version_dir[1] = dirver
553
554 dirs_uri = bb.fetch.encodeurl([ud.type, ud.host,
555 ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
556 bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
557
558 soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a"))
559 if not soup:
560 return version[1]
561
562 for line in soup.find_all('a', href=True):
563 s = dirver_regex.search(line['href'].strip("/"))
564 if s:
565 sver = s.group('ver')
566
567 # When prefix is part of the version directory it need to
568 # ensure that only version directory is used so remove previous
569 # directories if exists.
570 #
571 # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
572 # result is v2.5.
573 spfx = s.group('pfx').split('/')[-1]
574
575 version_dir_new = ['', sver, '']
576 if self._vercmp(version_dir, version_dir_new) <= 0:
577 dirver_new = spfx + sver
578 path = ud.path.replace(dirver, dirver_new, True) \
579 .split(package)[0]
580 uri = bb.fetch.encodeurl([ud.type, ud.host, path,
581 ud.user, ud.pswd, {}])
582
583 pupver = self._check_latest_version(uri,
584 package, package_regex, current_version, ud, d)
585 if pupver:
586 version[1] = pupver
587
588 version_dir = version_dir_new
589
590 return version[1]
591
592 def _init_regexes(self, package, ud, d):
593 """
594 Match as many patterns as possible such as:
595 gnome-common-2.20.0.tar.gz (most common format)
596 gtk+-2.90.1.tar.gz
597 xf86-input-synaptics-12.6.9.tar.gz
598 dri2proto-2.3.tar.gz
599 blktool_4.orig.tar.gz
600 libid3tag-0.15.1b.tar.gz
601 unzip552.tar.gz
602 icu4c-3_6-src.tgz
603 genext2fs_1.3.orig.tar.gz
604 gst-fluendo-mp3
605 """
606 # match most patterns which uses "-" as separator to version digits
607 pn_prefix1 = r"[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]"
608 # a loose pattern such as for unzip552.tar.gz
609 pn_prefix2 = r"[a-zA-Z]+"
610 # a loose pattern such as for 80325-quicky-0.4.tar.gz
611 pn_prefix3 = r"[0-9]+[-]?[a-zA-Z]+"
612 # Save the Package Name (pn) Regex for use later
613 pn_regex = r"(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
614
615 # match version
616 pver_regex = r"(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
617
618 # match arch
619 parch_regex = "-source|_all_"
620
621 # src.rpm extension was added only for rpm package. Can be removed if the rpm
622 # packaged will always be considered as having to be manually upgraded
623 psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
624
625 # match name, version and archive type of a package
626 package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
627 % (pn_regex, pver_regex, parch_regex, psuffix_regex))
628 self.suffix_regex_comp = re.compile(psuffix_regex)
629
630 # compile regex, can be specific by package or generic regex
631 pn_regex = d.getVar('UPSTREAM_CHECK_REGEX')
632 if pn_regex:
633 package_custom_regex_comp = re.compile(pn_regex)
634 else:
635 version = self._parse_path(package_regex_comp, package)
636 if version:
637 package_custom_regex_comp = re.compile(
638 r"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" %
639 (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex))
640 else:
641 package_custom_regex_comp = None
642
643 return package_custom_regex_comp
644
645 def latest_versionstring(self, ud, d):
646 """
647 Manipulate the URL and try to obtain the latest package version
648
649 sanity check to ensure same name and type.
650 """
651 if 'downloadfilename' in ud.parm:
652 package = ud.parm['downloadfilename']
653 else:
654 package = ud.path.split("/")[-1]
655 current_version = ['', d.getVar('PV'), '']
656
657 """possible to have no version in pkg name, such as spectrum-fw"""
658 if not re.search(r"\d+", package):
659 current_version[1] = re.sub('_', '.', current_version[1])
660 current_version[1] = re.sub('-', '.', current_version[1])
661 bb.debug(3, "latest_versionstring: no version found in %s" % package)
662 return (current_version[1], '')
663
664 package_regex = self._init_regexes(package, ud, d)
665 if package_regex is None:
666 bb.warn("latest_versionstring: package %s don't match pattern" % (package))
667 return ('', '')
668 bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
669
670 uri = ""
671 regex_uri = d.getVar("UPSTREAM_CHECK_URI")
672 if not regex_uri:
673 path = ud.path.split(package)[0]
674
675 # search for version matches on folders inside the path, like:
676 # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
677 dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
678 m = dirver_regex.findall(path)
679 if m:
680 pn = d.getVar('PN')
681 dirver = m[-1][0]
682
683 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
684 if not dirver_pn_regex.search(dirver):
685 return (self._check_latest_version_by_dir(dirver,
686 package, package_regex, current_version, ud, d), '')
687
688 uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
689 else:
690 uri = regex_uri
691
692 return (self._check_latest_version(uri, package, package_regex,
693 current_version, ud, d), '')
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2026-09-15 11:19:29 +0000