summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r--bitbake/lib/bb/fetch2/README57
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py561
-rw-r--r--bitbake/lib/bb/fetch2/az.py98
-rw-r--r--bitbake/lib/bb/fetch2/clearcase.py6
-rw-r--r--bitbake/lib/bb/fetch2/crate.py150
-rw-r--r--bitbake/lib/bb/fetch2/gcp.py102
-rw-r--r--bitbake/lib/bb/fetch2/git.py669
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py159
-rw-r--r--bitbake/lib/bb/fetch2/gomod.py273
-rw-r--r--bitbake/lib/bb/fetch2/hg.py1
-rw-r--r--bitbake/lib/bb/fetch2/local.py25
-rw-r--r--bitbake/lib/bb/fetch2/npm.py83
-rw-r--r--bitbake/lib/bb/fetch2/npmsw.py112
-rw-r--r--bitbake/lib/bb/fetch2/osc.py52
-rw-r--r--bitbake/lib/bb/fetch2/perforce.py2
-rw-r--r--bitbake/lib/bb/fetch2/s3.py43
-rw-r--r--bitbake/lib/bb/fetch2/sftp.py4
-rw-r--r--bitbake/lib/bb/fetch2/ssh.py50
-rw-r--r--bitbake/lib/bb/fetch2/svn.py15
-rw-r--r--bitbake/lib/bb/fetch2/wget.py263
20 files changed, 2051 insertions, 674 deletions
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README
new file mode 100644
index 0000000000..67b787ef47
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/README
@@ -0,0 +1,57 @@
1There are expectations of users of the fetcher code. This file attempts to document
2some of the constraints that are present. Some are obvious, some are less so. It is
3documented in the context of how OE uses it but the API calls are generic.
4
5a) network access for sources is only expected to happen in the do_fetch task.
6 This is not enforced or tested but is required so that we can:
7
8 i) audit the sources used (i.e. for license/manifest reasons)
9 ii) support offline builds with a suitable cache
10 iii) allow work to continue even with downtime upstream
11 iv) allow for changes upstream in incompatible ways
12 v) allow rebuilding of the software in X years time
13
14b) network access is not expected in do_unpack task.
15
16c) you can take DL_DIR and use it as a mirror for offline builds.
17
18d) access to the network is only made when explicitly configured in recipes
19 (e.g. use of AUTOREV, or use of git tags which change revision).
20
21e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it
22 will match in future exactly in a clean build with a new DL_DIR).
23 One specific pain point example are git tags. They can be replaced and change
24 so the git fetcher has to resolve them with the network. We use git revisions
25 where possible to avoid this and ensure determinism.
26
27f) network access is expected to work with the standard linux proxy variables
28 so that access behind firewalls works (the fetcher sets these in the
29 environment but only in the do_fetch tasks).
30
31g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV
32 git recipe might be ok but you can't expect to checkout a git tree.
33
34h) we need to provide revision information during parsing such that a version
35 for the recipe can be constructed.
36
37i) versions are expected to be able to increase in a way which sorts allowing
38 package feeds to operate (see PR server required for git revisions to sort).
39
40j) API to query for possible version upgrades of a url is highly desireable to
41 allow our automated upgrage code to function (it is implied this does always
42 have network access).
43
44k) Where fixes or changes to behaviour in the fetcher are made, we ask that
45 test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do
46 have fairly extensive test coverage of the fetcher as it is the only way
47 to track all of its corner cases, it still doesn't give entire coverage
48 though sadly.
49
50l) If using tools during parse time, they will have to be in ASSUME_PROVIDED
51 in OE's context as we can't build git-native, then parse a recipe and use
52 git ls-remote.
53
54Not all fetchers support all features, autorev is optional and doesn't make
55sense for some. Upgrade detection means different things in different contexts
56too.
57
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 19169d780f..0ad987c596 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -23,17 +23,18 @@ import collections
23import subprocess 23import subprocess
24import pickle 24import pickle
25import errno 25import errno
26import bb.persist_data, bb.utils 26import bb.utils
27import bb.checksum 27import bb.checksum
28import bb.process 28import bb.process
29import bb.event 29import bb.event
30 30
31__version__ = "2" 31__version__ = "2"
32_checksum_cache = bb.checksum.FileChecksumCache() 32_checksum_cache = bb.checksum.FileChecksumCache()
33_revisions_cache = bb.checksum.RevisionsCache()
33 34
34logger = logging.getLogger("BitBake.Fetcher") 35logger = logging.getLogger("BitBake.Fetcher")
35 36
36CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] 37CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ]
37SHOWN_CHECKSUM_LIST = ["sha256"] 38SHOWN_CHECKSUM_LIST = ["sha256"]
38 39
39class BBFetchException(Exception): 40class BBFetchException(Exception):
@@ -113,7 +114,7 @@ class MissingParameterError(BBFetchException):
113 self.args = (missing, url) 114 self.args = (missing, url)
114 115
115class ParameterError(BBFetchException): 116class ParameterError(BBFetchException):
116 """Exception raised when a url cannot be proccessed due to invalid parameters.""" 117 """Exception raised when a url cannot be processed due to invalid parameters."""
117 def __init__(self, message, url): 118 def __init__(self, message, url):
118 msg = "URL: '%s' has invalid parameters. %s" % (url, message) 119 msg = "URL: '%s' has invalid parameters. %s" % (url, message)
119 self.url = url 120 self.url = url
@@ -182,7 +183,7 @@ class URI(object):
182 Some notes about relative URIs: while it's specified that 183 Some notes about relative URIs: while it's specified that
183 a URI beginning with <scheme>:// should either be directly 184 a URI beginning with <scheme>:// should either be directly
184 followed by a hostname or a /, the old URI handling of the 185 followed by a hostname or a /, the old URI handling of the
185 fetch2 library did not comform to this. Therefore, this URI 186 fetch2 library did not conform to this. Therefore, this URI
186 class has some kludges to make sure that URIs are parsed in 187 class has some kludges to make sure that URIs are parsed in
187 a way comforming to bitbake's current usage. This URI class 188 a way comforming to bitbake's current usage. This URI class
188 supports the following: 189 supports the following:
@@ -199,7 +200,7 @@ class URI(object):
199 file://hostname/absolute/path.diff (would be IETF compliant) 200 file://hostname/absolute/path.diff (would be IETF compliant)
200 201
201 Note that the last case only applies to a list of 202 Note that the last case only applies to a list of
202 "whitelisted" schemes (currently only file://), that requires 203 explicitly allowed schemes (currently only file://), that requires
203 its URIs to not have a network location. 204 its URIs to not have a network location.
204 """ 205 """
205 206
@@ -237,7 +238,7 @@ class URI(object):
237 # to RFC compliant URL format. E.g.: 238 # to RFC compliant URL format. E.g.:
238 # file://foo.diff -> file:foo.diff 239 # file://foo.diff -> file:foo.diff
239 if urlp.scheme in self._netloc_forbidden: 240 if urlp.scheme in self._netloc_forbidden:
240 uri = re.sub("(?<=:)//(?!/)", "", uri, 1) 241 uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1)
241 reparse = 1 242 reparse = 1
242 243
243 if reparse: 244 if reparse:
@@ -290,12 +291,12 @@ class URI(object):
290 291
291 def _param_str_split(self, string, elmdelim, kvdelim="="): 292 def _param_str_split(self, string, elmdelim, kvdelim="="):
292 ret = collections.OrderedDict() 293 ret = collections.OrderedDict()
293 for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: 294 for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
294 ret[k] = v 295 ret[k] = v
295 return ret 296 return ret
296 297
297 def _param_str_join(self, dict_, elmdelim, kvdelim="="): 298 def _param_str_join(self, dict_, elmdelim, kvdelim="="):
298 return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) 299 return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
299 300
300 @property 301 @property
301 def hostport(self): 302 def hostport(self):
@@ -352,6 +353,14 @@ def decodeurl(url):
352 user, password, parameters). 353 user, password, parameters).
353 """ 354 """
354 355
356 uri = URI(url)
357 path = uri.path if uri.path else "/"
358 return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params
359
360def decodemirrorurl(url):
361 """Decodes a mirror URL into the tokens (scheme, network location, path,
362 user, password, parameters).
363 """
355 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) 364 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
356 if not m: 365 if not m:
357 raise MalformedUrl(url) 366 raise MalformedUrl(url)
@@ -370,6 +379,9 @@ def decodeurl(url):
370 elif type.lower() == 'file': 379 elif type.lower() == 'file':
371 host = "" 380 host = ""
372 path = location 381 path = location
382 if user:
383 path = user + '@' + path
384 user = ""
373 else: 385 else:
374 host = location 386 host = location
375 path = "/" 387 path = "/"
@@ -388,7 +400,7 @@ def decodeurl(url):
388 if s: 400 if s:
389 if not '=' in s: 401 if not '=' in s:
390 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) 402 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
391 s1, s2 = s.split('=') 403 s1, s2 = s.split('=', 1)
392 p[s1] = s2 404 p[s1] = s2
393 405
394 return type, host, urllib.parse.unquote(path), user, pswd, p 406 return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -402,34 +414,37 @@ def encodeurl(decoded):
402 414
403 if not type: 415 if not type:
404 raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) 416 raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
405 url = '%s://' % type 417 uri = URI()
418 uri.scheme = type
406 if user and type != "file": 419 if user and type != "file":
407 url += "%s" % user 420 uri.username = user
408 if pswd: 421 if pswd:
409 url += ":%s" % pswd 422 uri.password = pswd
410 url += "@"
411 if host and type != "file": 423 if host and type != "file":
412 url += "%s" % host 424 uri.hostname = host
413 if path: 425 if path:
414 # Standardise path to ensure comparisons work 426 # Standardise path to ensure comparisons work
415 while '//' in path: 427 while '//' in path:
416 path = path.replace("//", "/") 428 path = path.replace("//", "/")
417 url += "%s" % urllib.parse.quote(path) 429 uri.path = path
430 if type == "file":
431 # Use old not IETF compliant style
432 uri.relative = False
418 if p: 433 if p:
419 for parm in p: 434 uri.params = p
420 url += ";%s=%s" % (parm, p[parm])
421 435
422 return url 436 return str(uri)
423 437
424def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): 438def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
425 if not ud.url or not uri_find or not uri_replace: 439 if not ud.url or not uri_find or not uri_replace:
426 logger.error("uri_replace: passed an undefined value, not replacing") 440 logger.error("uri_replace: passed an undefined value, not replacing")
427 return None 441 return None
428 uri_decoded = list(decodeurl(ud.url)) 442 uri_decoded = list(decodemirrorurl(ud.url))
429 uri_find_decoded = list(decodeurl(uri_find)) 443 uri_find_decoded = list(decodemirrorurl(uri_find))
430 uri_replace_decoded = list(decodeurl(uri_replace)) 444 uri_replace_decoded = list(decodemirrorurl(uri_replace))
431 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) 445 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
432 result_decoded = ['', '', '', '', '', {}] 446 result_decoded = ['', '', '', '', '', {}]
447 # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params
433 for loc, i in enumerate(uri_find_decoded): 448 for loc, i in enumerate(uri_find_decoded):
434 result_decoded[loc] = uri_decoded[loc] 449 result_decoded[loc] = uri_decoded[loc]
435 regexp = i 450 regexp = i
@@ -449,6 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
449 for l in replacements: 464 for l in replacements:
450 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) 465 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
451 result_decoded[loc][k] = uri_replace_decoded[loc][k] 466 result_decoded[loc][k] = uri_replace_decoded[loc][k]
467 elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
468 # User/password in the replacement is just a straight replacement
469 result_decoded[loc] = uri_replace_decoded[loc]
452 elif (re.match(regexp, uri_decoded[loc])): 470 elif (re.match(regexp, uri_decoded[loc])):
453 if not uri_replace_decoded[loc]: 471 if not uri_replace_decoded[loc]:
454 result_decoded[loc] = "" 472 result_decoded[loc] = ""
@@ -456,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
456 for k in replacements: 474 for k in replacements:
457 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) 475 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
458 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) 476 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
459 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) 477 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
460 if loc == 2: 478 if loc == 2:
461 # Handle path manipulations 479 # Handle path manipulations
462 basename = None 480 basename = None
@@ -465,10 +483,18 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
465 basename = os.path.basename(mirrortarball) 483 basename = os.path.basename(mirrortarball)
466 # Kill parameters, they make no sense for mirror tarballs 484 # Kill parameters, they make no sense for mirror tarballs
467 uri_decoded[5] = {} 485 uri_decoded[5] = {}
486 uri_find_decoded[5] = {}
468 elif ud.localpath and ud.method.supports_checksum(ud): 487 elif ud.localpath and ud.method.supports_checksum(ud):
469 basename = os.path.basename(ud.localpath) 488 basename = os.path.basename(ud.localpath)
470 if basename and not result_decoded[loc].endswith(basename): 489 if basename:
471 result_decoded[loc] = os.path.join(result_decoded[loc], basename) 490 uri_basename = os.path.basename(uri_decoded[loc])
491 # Prefix with a slash as a sentinel in case
492 # result_decoded[loc] does not contain one.
493 path = "/" + result_decoded[loc]
494 if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename):
495 result_decoded[loc] = path[1:-len(uri_basename)] + basename
496 elif not path.endswith("/" + basename):
497 result_decoded[loc] = os.path.join(path[1:], basename)
472 else: 498 else:
473 return None 499 return None
474 result = encodeurl(result_decoded) 500 result = encodeurl(result_decoded)
@@ -481,18 +507,23 @@ methods = []
481urldata_cache = {} 507urldata_cache = {}
482saved_headrevs = {} 508saved_headrevs = {}
483 509
484def fetcher_init(d): 510def fetcher_init(d, servercontext=True):
485 """ 511 """
486 Called to initialize the fetchers once the configuration data is known. 512 Called to initialize the fetchers once the configuration data is known.
487 Calls before this must not hit the cache. 513 Calls before this must not hit the cache.
488 """ 514 """
489 515
490 revs = bb.persist_data.persist('BB_URI_HEADREVS', d) 516 _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
517 _revisions_cache.init_cache(d.getVar("BB_CACHEDIR"))
518
519 if not servercontext:
520 return
521
491 try: 522 try:
492 # fetcher_init is called multiple times, so make sure we only save the 523 # fetcher_init is called multiple times, so make sure we only save the
493 # revs the first time it is called. 524 # revs the first time it is called.
494 if not bb.fetch2.saved_headrevs: 525 if not bb.fetch2.saved_headrevs:
495 bb.fetch2.saved_headrevs = dict(revs) 526 bb.fetch2.saved_headrevs = _revisions_cache.get_revs()
496 except: 527 except:
497 pass 528 pass
498 529
@@ -502,11 +533,10 @@ def fetcher_init(d):
502 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) 533 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
503 elif srcrev_policy == "clear": 534 elif srcrev_policy == "clear":
504 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) 535 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
505 revs.clear() 536 _revisions_cache.clear_cache()
506 else: 537 else:
507 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) 538 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
508 539
509 _checksum_cache.init_cache(d)
510 540
511 for m in methods: 541 for m in methods:
512 if hasattr(m, "init"): 542 if hasattr(m, "init"):
@@ -514,9 +544,11 @@ def fetcher_init(d):
514 544
515def fetcher_parse_save(): 545def fetcher_parse_save():
516 _checksum_cache.save_extras() 546 _checksum_cache.save_extras()
547 _revisions_cache.save_extras()
517 548
518def fetcher_parse_done(): 549def fetcher_parse_done():
519 _checksum_cache.save_merge() 550 _checksum_cache.save_merge()
551 _revisions_cache.save_merge()
520 552
521def fetcher_compare_revisions(d): 553def fetcher_compare_revisions(d):
522 """ 554 """
@@ -524,7 +556,7 @@ def fetcher_compare_revisions(d):
524 when bitbake was started and return true if they have changed. 556 when bitbake was started and return true if they have changed.
525 """ 557 """
526 558
527 headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) 559 headrevs = _revisions_cache.get_revs()
528 return headrevs != bb.fetch2.saved_headrevs 560 return headrevs != bb.fetch2.saved_headrevs
529 561
530def mirror_from_string(data): 562def mirror_from_string(data):
@@ -534,7 +566,7 @@ def mirror_from_string(data):
534 bb.warn('Invalid mirror data %s, should have paired members.' % data) 566 bb.warn('Invalid mirror data %s, should have paired members.' % data)
535 return list(zip(*[iter(mirrors)]*2)) 567 return list(zip(*[iter(mirrors)]*2))
536 568
537def verify_checksum(ud, d, precomputed={}): 569def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
538 """ 570 """
539 verify the MD5 and SHA256 checksum for downloaded src 571 verify the MD5 and SHA256 checksum for downloaded src
540 572
@@ -548,20 +580,25 @@ def verify_checksum(ud, d, precomputed={}):
548 file against those in the recipe each time, rather than only after 580 file against those in the recipe each time, rather than only after
549 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. 581 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
550 """ 582 """
551
552 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 583 if ud.ignore_checksums or not ud.method.supports_checksum(ud):
553 return {} 584 return {}
554 585
586 if localpath is None:
587 localpath = ud.localpath
588
555 def compute_checksum_info(checksum_id): 589 def compute_checksum_info(checksum_id):
556 checksum_name = getattr(ud, "%s_name" % checksum_id) 590 checksum_name = getattr(ud, "%s_name" % checksum_id)
557 591
558 if checksum_id in precomputed: 592 if checksum_id in precomputed:
559 checksum_data = precomputed[checksum_id] 593 checksum_data = precomputed[checksum_id]
560 else: 594 else:
561 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) 595 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
562 596
563 checksum_expected = getattr(ud, "%s_expected" % checksum_id) 597 checksum_expected = getattr(ud, "%s_expected" % checksum_id)
564 598
599 if checksum_expected == '':
600 checksum_expected = None
601
565 return { 602 return {
566 "id": checksum_id, 603 "id": checksum_id,
567 "name": checksum_name, 604 "name": checksum_name,
@@ -581,17 +618,13 @@ def verify_checksum(ud, d, precomputed={}):
581 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] 618 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
582 619
583 # If no checksum has been provided 620 # If no checksum has been provided
584 if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): 621 if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
585 messages = [] 622 messages = []
586 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 623 strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
587 624
588 # If strict checking enabled and neither sum defined, raise error 625 # If strict checking enabled and neither sum defined, raise error
589 if strict == "1": 626 if strict == "1":
590 messages.append("No checksum specified for '%s', please add at " \ 627 raise NoChecksumError("\n".join(checksum_lines))
591 "least one to the recipe:" % ud.localpath)
592 messages.extend(checksum_lines)
593 logger.error("\n".join(messages))
594 raise NoChecksumError("Missing SRC_URI checksum", ud.url)
595 628
596 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) 629 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
597 630
@@ -612,8 +645,8 @@ def verify_checksum(ud, d, precomputed={}):
612 645
613 for ci in checksum_infos: 646 for ci in checksum_infos:
614 if ci["expected"] and ci["expected"] != ci["data"]: 647 if ci["expected"] and ci["expected"] != ci["data"]:
615 messages.append("File: '%s' has %s checksum %s when %s was " \ 648 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
616 "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) 649 "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
617 bad_checksum = ci["data"] 650 bad_checksum = ci["data"]
618 651
619 if bad_checksum: 652 if bad_checksum:
@@ -731,13 +764,16 @@ def subprocess_setup():
731 # SIGPIPE errors are known issues with gzip/bash 764 # SIGPIPE errors are known issues with gzip/bash
732 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 765 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
733 766
734def get_autorev(d): 767def mark_recipe_nocache(d):
735 # only not cache src rev in autorev case
736 if d.getVar('BB_SRCREV_POLICY') != "cache": 768 if d.getVar('BB_SRCREV_POLICY') != "cache":
737 d.setVar('BB_DONT_CACHE', '1') 769 d.setVar('BB_DONT_CACHE', '1')
770
771def get_autorev(d):
772 mark_recipe_nocache(d)
773 d.setVar("__BBAUTOREV_SEEN", True)
738 return "AUTOINC" 774 return "AUTOINC"
739 775
740def get_srcrev(d, method_name='sortable_revision'): 776def _get_srcrev(d, method_name='sortable_revision'):
741 """ 777 """
742 Return the revision string, usually for use in the version string (PV) of the current package 778 Return the revision string, usually for use in the version string (PV) of the current package
743 Most packages usually only have one SCM so we just pass on the call. 779 Most packages usually only have one SCM so we just pass on the call.
@@ -751,23 +787,34 @@ def get_srcrev(d, method_name='sortable_revision'):
751 that fetcher provides a method with the given name and the same signature as sortable_revision. 787 that fetcher provides a method with the given name and the same signature as sortable_revision.
752 """ 788 """
753 789
790 d.setVar("__BBSRCREV_SEEN", "1")
791 recursion = d.getVar("__BBINSRCREV")
792 if recursion:
793 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
794 d.setVar("__BBINSRCREV", True)
795
754 scms = [] 796 scms = []
797 revs = []
755 fetcher = Fetch(d.getVar('SRC_URI').split(), d) 798 fetcher = Fetch(d.getVar('SRC_URI').split(), d)
756 urldata = fetcher.ud 799 urldata = fetcher.ud
757 for u in urldata: 800 for u in urldata:
758 if urldata[u].method.supports_srcrev(): 801 if urldata[u].method.supports_srcrev():
759 scms.append(u) 802 scms.append(u)
760 803
761 if len(scms) == 0: 804 if not scms:
762 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") 805 d.delVar("__BBINSRCREV")
806 return "", revs
763 807
764 if len(scms) == 1 and len(urldata[scms[0]].names) == 1: 808
765 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) 809 if len(scms) == 1:
810 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
811 revs.append(rev)
766 if len(rev) > 10: 812 if len(rev) > 10:
767 rev = rev[:10] 813 rev = rev[:10]
814 d.delVar("__BBINSRCREV")
768 if autoinc: 815 if autoinc:
769 return "AUTOINC+" + rev 816 return "AUTOINC+" + rev, revs
770 return rev 817 return rev, revs
771 818
772 # 819 #
773 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT 820 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -781,12 +828,12 @@ def get_srcrev(d, method_name='sortable_revision'):
781 seenautoinc = False 828 seenautoinc = False
782 for scm in scms: 829 for scm in scms:
783 ud = urldata[scm] 830 ud = urldata[scm]
784 for name in ud.names: 831 autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
785 autoinc, rev = getattr(ud.method, method_name)(ud, d, name) 832 revs.append(rev)
786 seenautoinc = seenautoinc or autoinc 833 seenautoinc = seenautoinc or autoinc
787 if len(rev) > 10: 834 if len(rev) > 10:
788 rev = rev[:10] 835 rev = rev[:10]
789 name_to_rev[name] = rev 836 name_to_rev[ud.name] = rev
790 # Replace names by revisions in the SRCREV_FORMAT string. The approach used 837 # Replace names by revisions in the SRCREV_FORMAT string. The approach used
791 # here can handle names being prefixes of other names and names appearing 838 # here can handle names being prefixes of other names and names appearing
792 # as substrings in revisions (in which case the name should not be 839 # as substrings in revisions (in which case the name should not be
@@ -799,12 +846,71 @@ def get_srcrev(d, method_name='sortable_revision'):
799 if seenautoinc: 846 if seenautoinc:
800 format = "AUTOINC+" + format 847 format = "AUTOINC+" + format
801 848
802 return format 849 d.delVar("__BBINSRCREV")
850 return format, revs
851
852def get_hashvalue(d, method_name='sortable_revision'):
853 pkgv, revs = _get_srcrev(d, method_name=method_name)
854 return " ".join(revs)
855
856def get_pkgv_string(d, method_name='sortable_revision'):
857 pkgv, revs = _get_srcrev(d, method_name=method_name)
858 return pkgv
859
860def get_srcrev(d, method_name='sortable_revision'):
861 pkgv, revs = _get_srcrev(d, method_name=method_name)
862 if not pkgv:
863 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
864 return pkgv
803 865
804def localpath(url, d): 866def localpath(url, d):
805 fetcher = bb.fetch2.Fetch([url], d) 867 fetcher = bb.fetch2.Fetch([url], d)
806 return fetcher.localpath(url) 868 return fetcher.localpath(url)
807 869
870# Need to export PATH as binary could be in metadata paths
871# rather than host provided
872# Also include some other variables.
873FETCH_EXPORT_VARS = ['HOME', 'PATH',
874 'HTTP_PROXY', 'http_proxy',
875 'HTTPS_PROXY', 'https_proxy',
876 'FTP_PROXY', 'ftp_proxy',
877 'FTPS_PROXY', 'ftps_proxy',
878 'NO_PROXY', 'no_proxy',
879 'ALL_PROXY', 'all_proxy',
880 'GIT_PROXY_COMMAND',
881 'GIT_SSH',
882 'GIT_SSH_COMMAND',
883 'GIT_SSL_CAINFO',
884 'GIT_SMART_HTTP',
885 'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
886 'SOCKS5_USER', 'SOCKS5_PASSWD',
887 'DBUS_SESSION_BUS_ADDRESS',
888 'P4CONFIG',
889 'SSL_CERT_FILE',
890 'NODE_EXTRA_CA_CERTS',
891 'AWS_PROFILE',
892 'AWS_ACCESS_KEY_ID',
893 'AWS_SECRET_ACCESS_KEY',
894 'AWS_ROLE_ARN',
895 'AWS_WEB_IDENTITY_TOKEN_FILE',
896 'AWS_DEFAULT_REGION',
897 'AWS_SESSION_TOKEN',
898 'GIT_CACHE_PATH',
899 'REMOTE_CONTAINERS_IPC',
900 'GITHUB_TOKEN',
901 'SSL_CERT_DIR']
902
903def get_fetcher_environment(d):
904 newenv = {}
905 origenv = d.getVar("BB_ORIGENV")
906 for name in bb.fetch2.FETCH_EXPORT_VARS:
907 value = d.getVar(name)
908 if not value and origenv:
909 value = origenv.getVar(name)
910 if value:
911 newenv[name] = value
912 return newenv
913
808def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): 914def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
809 """ 915 """
810 Run cmd returning the command output 916 Run cmd returning the command output
@@ -813,25 +919,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
813 Optionally remove the files/directories listed in cleanup upon failure 919 Optionally remove the files/directories listed in cleanup upon failure
814 """ 920 """
815 921
816 # Need to export PATH as binary could be in metadata paths 922 exportvars = FETCH_EXPORT_VARS
817 # rather than host provided
818 # Also include some other variables.
819 # FIXME: Should really include all export varaiables?
820 exportvars = ['HOME', 'PATH',
821 'HTTP_PROXY', 'http_proxy',
822 'HTTPS_PROXY', 'https_proxy',
823 'FTP_PROXY', 'ftp_proxy',
824 'FTPS_PROXY', 'ftps_proxy',
825 'NO_PROXY', 'no_proxy',
826 'ALL_PROXY', 'all_proxy',
827 'GIT_PROXY_COMMAND',
828 'GIT_SSH',
829 'GIT_SSL_CAINFO',
830 'GIT_SMART_HTTP',
831 'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
832 'SOCKS5_USER', 'SOCKS5_PASSWD',
833 'DBUS_SESSION_BUS_ADDRESS',
834 'P4CONFIG']
835 923
836 if not cleanup: 924 if not cleanup:
837 cleanup = [] 925 cleanup = []
@@ -868,14 +956,17 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
868 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) 956 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
869 success = True 957 success = True
870 except bb.process.NotFoundError as e: 958 except bb.process.NotFoundError as e:
871 error_message = "Fetch command %s" % (e.command) 959 error_message = "Fetch command %s not found" % (e.command)
872 except bb.process.ExecutionError as e: 960 except bb.process.ExecutionError as e:
873 if e.stdout: 961 if e.stdout:
874 output = "output:\n%s\n%s" % (e.stdout, e.stderr) 962 output = "output:\n%s\n%s" % (e.stdout, e.stderr)
875 elif e.stderr: 963 elif e.stderr:
876 output = "output:\n%s" % e.stderr 964 output = "output:\n%s" % e.stderr
877 else: 965 else:
878 output = "no output" 966 if log:
967 output = "see logfile for output"
968 else:
969 output = "no output"
879 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) 970 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
880 except bb.process.CmdError as e: 971 except bb.process.CmdError as e:
881 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) 972 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
@@ -937,6 +1028,7 @@ def build_mirroruris(origud, mirrors, ld):
937 1028
938 try: 1029 try:
939 newud = FetchData(newuri, ld) 1030 newud = FetchData(newuri, ld)
1031 newud.ignore_checksums = True
940 newud.setup_localpath(ld) 1032 newud.setup_localpath(ld)
941 except bb.fetch2.BBFetchException as e: 1033 except bb.fetch2.BBFetchException as e:
942 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) 1034 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1000,6 +1092,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
1000 # If that tarball is a local file:// we need to provide a symlink to it 1092 # If that tarball is a local file:// we need to provide a symlink to it
1001 dldir = ld.getVar("DL_DIR") 1093 dldir = ld.getVar("DL_DIR")
1002 1094
1095 if bb.utils.to_boolean(ld.getVar("BB_FETCH_PREMIRRORONLY")):
1096 ld = ld.createCopy()
1097 ld.setVar("BB_NO_NETWORK", "1")
1098
1003 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): 1099 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
1004 # Create donestamp in old format to avoid triggering a re-download 1100 # Create donestamp in old format to avoid triggering a re-download
1005 if ud.donestamp: 1101 if ud.donestamp:
@@ -1021,7 +1117,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
1021 origud.method.build_mirror_data(origud, ld) 1117 origud.method.build_mirror_data(origud, ld)
1022 return origud.localpath 1118 return origud.localpath
1023 # Otherwise the result is a local file:// and we symlink to it 1119 # Otherwise the result is a local file:// and we symlink to it
1024 ensure_symlink(ud.localpath, origud.localpath) 1120 # This may also be a link to a shallow archive
1121 # When using shallow mode, add a symlink to the original fullshallow
1122 # path to ensure a valid symlink even in the `PREMIRRORS` case
1123 origud.method.update_mirror_links(ud, origud)
1025 update_stamp(origud, ld) 1124 update_stamp(origud, ld)
1026 return ud.localpath 1125 return ud.localpath
1027 1126
@@ -1046,7 +1145,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
1046 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) 1145 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
1047 logger.debug(str(e)) 1146 logger.debug(str(e))
1048 try: 1147 try:
1049 ud.method.clean(ud, ld) 1148 if ud.method.cleanup_upon_failure():
1149 ud.method.clean(ud, ld)
1050 except UnboundLocalError: 1150 except UnboundLocalError:
1051 pass 1151 pass
1052 return False 1152 return False
@@ -1054,23 +1154,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
1054 if ud.lockfile and ud.lockfile != origud.lockfile: 1154 if ud.lockfile and ud.lockfile != origud.lockfile:
1055 bb.utils.unlockfile(lf) 1155 bb.utils.unlockfile(lf)
1056 1156
1057
1058def ensure_symlink(target, link_name):
1059 if not os.path.exists(link_name):
1060 if os.path.islink(link_name):
1061 # Broken symbolic link
1062 os.unlink(link_name)
1063
1064 # In case this is executing without any file locks held (as is
1065 # the case for file:// URLs), two tasks may end up here at the
1066 # same time, in which case we do not want the second task to
1067 # fail when the link has already been created by the first task.
1068 try:
1069 os.symlink(target, link_name)
1070 except FileExistsError:
1071 pass
1072
1073
1074def try_mirrors(fetch, d, origud, mirrors, check = False): 1157def try_mirrors(fetch, d, origud, mirrors, check = False):
1075 """ 1158 """
1076 Try to use a mirrored version of the sources. 1159 Try to use a mirrored version of the sources.
@@ -1099,7 +1182,7 @@ def trusted_network(d, url):
1099 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 1182 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
1100 return True 1183 return True
1101 1184
1102 pkgname = d.expand(d.getVar('PN', False)) 1185 pkgname = d.getVar('PN')
1103 trusted_hosts = None 1186 trusted_hosts = None
1104 if pkgname: 1187 if pkgname:
1105 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) 1188 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
@@ -1140,11 +1223,11 @@ def srcrev_internal_helper(ud, d, name):
1140 pn = d.getVar("PN") 1223 pn = d.getVar("PN")
1141 attempts = [] 1224 attempts = []
1142 if name != '' and pn: 1225 if name != '' and pn:
1143 attempts.append("SRCREV_%s_pn-%s" % (name, pn)) 1226 attempts.append("SRCREV_%s:pn-%s" % (name, pn))
1144 if name != '': 1227 if name != '':
1145 attempts.append("SRCREV_%s" % name) 1228 attempts.append("SRCREV_%s" % name)
1146 if pn: 1229 if pn:
1147 attempts.append("SRCREV_pn-%s" % pn) 1230 attempts.append("SRCREV:pn-%s" % pn)
1148 attempts.append("SRCREV") 1231 attempts.append("SRCREV")
1149 1232
1150 for a in attempts: 1233 for a in attempts:
@@ -1152,23 +1235,21 @@ def srcrev_internal_helper(ud, d, name):
1152 if srcrev and srcrev != "INVALID": 1235 if srcrev and srcrev != "INVALID":
1153 break 1236 break
1154 1237
1155 if 'rev' in ud.parm and 'tag' in ud.parm: 1238 if 'rev' in ud.parm:
1156 raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) 1239 parmrev = ud.parm['rev']
1157
1158 if 'rev' in ud.parm or 'tag' in ud.parm:
1159 if 'rev' in ud.parm:
1160 parmrev = ud.parm['rev']
1161 else:
1162 parmrev = ud.parm['tag']
1163 if srcrev == "INVALID" or not srcrev: 1240 if srcrev == "INVALID" or not srcrev:
1164 return parmrev 1241 return parmrev
1165 if srcrev != parmrev: 1242 if srcrev != parmrev:
1166 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) 1243 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev))
1167 return parmrev 1244 return parmrev
1168 1245
1246 if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev):
1247 return ud.parm['tag']
1248
1169 if srcrev == "INVALID" or not srcrev: 1249 if srcrev == "INVALID" or not srcrev:
1170 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) 1250 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
1171 if srcrev == "AUTOINC": 1251 if srcrev == "AUTOINC":
1252 d.setVar("__BBAUTOREV_ACTED_UPON", True)
1172 srcrev = ud.method.latest_revision(ud, d, name) 1253 srcrev = ud.method.latest_revision(ud, d, name)
1173 1254
1174 return srcrev 1255 return srcrev
@@ -1180,23 +1261,21 @@ def get_checksum_file_list(d):
1180 SRC_URI as a space-separated string 1261 SRC_URI as a space-separated string
1181 """ 1262 """
1182 fetch = Fetch([], d, cache = False, localonly = True) 1263 fetch = Fetch([], d, cache = False, localonly = True)
1183
1184 dl_dir = d.getVar('DL_DIR')
1185 filelist = [] 1264 filelist = []
1186 for u in fetch.urls: 1265 for u in fetch.urls:
1187 ud = fetch.ud[u] 1266 ud = fetch.ud[u]
1188
1189 if ud and isinstance(ud.method, local.Local): 1267 if ud and isinstance(ud.method, local.Local):
1190 paths = ud.method.localpaths(ud, d) 1268 found = False
1269 paths = ud.method.localfile_searchpaths(ud, d)
1191 for f in paths: 1270 for f in paths:
1192 pth = ud.decodedurl 1271 pth = ud.path
1193 if f.startswith(dl_dir): 1272 if os.path.exists(f):
1194 # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else 1273 found = True
1195 if os.path.exists(f):
1196 bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
1197 else:
1198 bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
1199 filelist.append(f + ":" + str(os.path.exists(f))) 1274 filelist.append(f + ":" + str(os.path.exists(f)))
1275 if not found:
1276 bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
1277 "\nThe following paths were searched:"
1278 "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
1200 1279
1201 return " ".join(filelist) 1280 return " ".join(filelist)
1202 1281
@@ -1234,28 +1313,28 @@ class FetchData(object):
1234 self.setup = False 1313 self.setup = False
1235 1314
1236 def configure_checksum(checksum_id): 1315 def configure_checksum(checksum_id):
1316 checksum_plain_name = "%ssum" % checksum_id
1237 if "name" in self.parm: 1317 if "name" in self.parm:
1238 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) 1318 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
1239 else: 1319 else:
1240 checksum_name = "%ssum" % checksum_id 1320 checksum_name = checksum_plain_name
1241
1242 setattr(self, "%s_name" % checksum_id, checksum_name)
1243 1321
1244 if checksum_name in self.parm: 1322 if checksum_name in self.parm:
1245 checksum_expected = self.parm[checksum_name] 1323 checksum_expected = self.parm[checksum_name]
1246 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: 1324 elif checksum_plain_name in self.parm:
1325 checksum_expected = self.parm[checksum_plain_name]
1326 checksum_name = checksum_plain_name
1327 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]:
1247 checksum_expected = None 1328 checksum_expected = None
1248 else: 1329 else:
1249 checksum_expected = d.getVarFlag("SRC_URI", checksum_name) 1330 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
1250 1331
1332 setattr(self, "%s_name" % checksum_id, checksum_name)
1251 setattr(self, "%s_expected" % checksum_id, checksum_expected) 1333 setattr(self, "%s_expected" % checksum_id, checksum_expected)
1252 1334
1253 for checksum_id in CHECKSUM_LIST: 1335 self.name = self.parm.get("name",'default')
1254 configure_checksum(checksum_id) 1336 if "," in self.name:
1255 1337 raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
1256 self.ignore_checksums = False
1257
1258 self.names = self.parm.get("name",'default').split(',')
1259 1338
1260 self.method = None 1339 self.method = None
1261 for m in methods: 1340 for m in methods:
@@ -1276,6 +1355,11 @@ class FetchData(object):
1276 if hasattr(self.method, "urldata_init"): 1355 if hasattr(self.method, "urldata_init"):
1277 self.method.urldata_init(self, d) 1356 self.method.urldata_init(self, d)
1278 1357
1358 for checksum_id in CHECKSUM_LIST:
1359 configure_checksum(checksum_id)
1360
1361 self.ignore_checksums = False
1362
1279 if "localpath" in self.parm: 1363 if "localpath" in self.parm:
1280 # if user sets localpath for file, use it instead. 1364 # if user sets localpath for file, use it instead.
1281 self.localpath = self.parm["localpath"] 1365 self.localpath = self.parm["localpath"]
@@ -1302,13 +1386,7 @@ class FetchData(object):
1302 self.lockfile = basepath + '.lock' 1386 self.lockfile = basepath + '.lock'
1303 1387
1304 def setup_revisions(self, d): 1388 def setup_revisions(self, d):
1305 self.revisions = {} 1389 self.revision = srcrev_internal_helper(self, d, self.name)
1306 for name in self.names:
1307 self.revisions[name] = srcrev_internal_helper(self, d, name)
1308
1309 # add compatibility code for non name specified case
1310 if len(self.names) == 1:
1311 self.revision = self.revisions[self.names[0]]
1312 1390
1313 def setup_localpath(self, d): 1391 def setup_localpath(self, d):
1314 if not self.localpath: 1392 if not self.localpath:
@@ -1355,6 +1433,9 @@ class FetchMethod(object):
1355 Is localpath something that can be represented by a checksum? 1433 Is localpath something that can be represented by a checksum?
1356 """ 1434 """
1357 1435
1436 # We cannot compute checksums for None
1437 if urldata.localpath is None:
1438 return False
1358 # We cannot compute checksums for directories 1439 # We cannot compute checksums for directories
1359 if os.path.isdir(urldata.localpath): 1440 if os.path.isdir(urldata.localpath):
1360 return False 1441 return False
@@ -1367,6 +1448,12 @@ class FetchMethod(object):
1367 """ 1448 """
1368 return False 1449 return False
1369 1450
1451 def cleanup_upon_failure(self):
1452 """
1453 When a fetch fails, should clean() be called?
1454 """
1455 return True
1456
1370 def verify_donestamp(self, ud, d): 1457 def verify_donestamp(self, ud, d):
1371 """ 1458 """
1372 Verify the donestamp file 1459 Verify the donestamp file
@@ -1427,37 +1514,40 @@ class FetchMethod(object):
1427 (file, urldata.parm.get('unpack'))) 1514 (file, urldata.parm.get('unpack')))
1428 1515
1429 base, ext = os.path.splitext(file) 1516 base, ext = os.path.splitext(file)
1430 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']: 1517 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']:
1431 efile = os.path.join(rootdir, os.path.basename(base)) 1518 efile = os.path.join(rootdir, os.path.basename(base))
1432 else: 1519 else:
1433 efile = file 1520 efile = file
1434 cmd = None 1521 cmd = None
1435 1522
1436 if unpack: 1523 if unpack:
1524 tar_cmd = 'tar --extract --no-same-owner'
1525 if 'striplevel' in urldata.parm:
1526 tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel']
1437 if file.endswith('.tar'): 1527 if file.endswith('.tar'):
1438 cmd = 'tar x --no-same-owner -f %s' % file 1528 cmd = '%s -f %s' % (tar_cmd, file)
1439 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 1529 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
1440 cmd = 'tar xz --no-same-owner -f %s' % file 1530 cmd = '%s -z -f %s' % (tar_cmd, file)
1441 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): 1531 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
1442 cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file 1532 cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd)
1443 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): 1533 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
1444 cmd = 'gzip -dc %s > %s' % (file, efile) 1534 cmd = 'gzip -dc %s > %s' % (file, efile)
1445 elif file.endswith('.bz2'): 1535 elif file.endswith('.bz2'):
1446 cmd = 'bzip2 -dc %s > %s' % (file, efile) 1536 cmd = 'bzip2 -dc %s > %s' % (file, efile)
1447 elif file.endswith('.txz') or file.endswith('.tar.xz'): 1537 elif file.endswith('.txz') or file.endswith('.tar.xz'):
1448 cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file 1538 cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd)
1449 elif file.endswith('.xz'): 1539 elif file.endswith('.xz'):
1450 cmd = 'xz -dc %s > %s' % (file, efile) 1540 cmd = 'xz -dc %s > %s' % (file, efile)
1451 elif file.endswith('.tar.lz'): 1541 elif file.endswith('.tar.lz'):
1452 cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file 1542 cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd)
1453 elif file.endswith('.lz'): 1543 elif file.endswith('.lz'):
1454 cmd = 'lzip -dc %s > %s' % (file, efile) 1544 cmd = 'lzip -dc %s > %s' % (file, efile)
1455 elif file.endswith('.tar.7z'): 1545 elif file.endswith('.tar.7z'):
1456 cmd = '7z x -so %s | tar x --no-same-owner -f -' % file 1546 cmd = '7z x -so %s | %s -f -' % (file, tar_cmd)
1457 elif file.endswith('.7z'): 1547 elif file.endswith('.7z'):
1458 cmd = '7za x -y %s 1>/dev/null' % file 1548 cmd = '7za x -y %s 1>/dev/null' % file
1459 elif file.endswith('.tzst') or file.endswith('.tar.zst'): 1549 elif file.endswith('.tzst') or file.endswith('.tar.zst'):
1460 cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file 1550 cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd)
1461 elif file.endswith('.zst'): 1551 elif file.endswith('.zst'):
1462 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) 1552 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
1463 elif file.endswith('.zip') or file.endswith('.jar'): 1553 elif file.endswith('.zip') or file.endswith('.jar'):
@@ -1483,14 +1573,14 @@ class FetchMethod(object):
1483 datafile = None 1573 datafile = None
1484 if output: 1574 if output:
1485 for line in output.decode().splitlines(): 1575 for line in output.decode().splitlines():
1486 if line.startswith('data.tar.'): 1576 if line.startswith('data.tar.') or line == 'data.tar':
1487 datafile = line 1577 datafile = line
1488 break 1578 break
1489 else: 1579 else:
1490 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) 1580 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar* file", urldata.url)
1491 else: 1581 else:
1492 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) 1582 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
1493 cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile) 1583 cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
1494 1584
1495 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd 1585 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
1496 if 'subdir' in urldata.parm: 1586 if 'subdir' in urldata.parm:
@@ -1506,6 +1596,7 @@ class FetchMethod(object):
1506 unpackdir = rootdir 1596 unpackdir = rootdir
1507 1597
1508 if not unpack or not cmd: 1598 if not unpack or not cmd:
1599 urldata.unpack_tracer.unpack("file-copy", unpackdir)
1509 # If file == dest, then avoid any copies, as we already put the file into dest! 1600 # If file == dest, then avoid any copies, as we already put the file into dest!
1510 dest = os.path.join(unpackdir, os.path.basename(file)) 1601 dest = os.path.join(unpackdir, os.path.basename(file))
1511 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): 1602 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -1519,7 +1610,9 @@ class FetchMethod(object):
1519 if urlpath.find("/") != -1: 1610 if urlpath.find("/") != -1:
1520 destdir = urlpath.rsplit("/", 1)[0] + '/' 1611 destdir = urlpath.rsplit("/", 1)[0] + '/'
1521 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) 1612 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
1522 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) 1613 cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir)
1614 else:
1615 urldata.unpack_tracer.unpack("archive-extract", unpackdir)
1523 1616
1524 if not cmd: 1617 if not cmd:
1525 return 1618 return
@@ -1546,6 +1639,28 @@ class FetchMethod(object):
1546 """ 1639 """
1547 bb.utils.remove(urldata.localpath) 1640 bb.utils.remove(urldata.localpath)
1548 1641
1642 def ensure_symlink(self, target, link_name):
1643 if not os.path.exists(link_name):
1644 dirname = os.path.dirname(link_name)
1645 bb.utils.mkdirhier(dirname)
1646 if os.path.islink(link_name):
1647 # Broken symbolic link
1648 os.unlink(link_name)
1649
1650 # In case this is executing without any file locks held (as is
1651 # the case for file:// URLs), two tasks may end up here at the
1652 # same time, in which case we do not want the second task to
1653 # fail when the link has already been created by the first task.
1654 try:
1655 os.symlink(target, link_name)
1656 except FileExistsError:
1657 pass
1658
1659 def update_mirror_links(self, ud, origud):
1660 # For local file:// results, create a symlink to them
1661 # This may also be a link to a shallow archive
1662 self.ensure_symlink(ud.localpath, origud.localpath)
1663
1549 def try_premirror(self, urldata, d): 1664 def try_premirror(self, urldata, d):
1550 """ 1665 """
1551 Should premirrors be used? 1666 Should premirrors be used?
@@ -1573,13 +1688,13 @@ class FetchMethod(object):
1573 if not hasattr(self, "_latest_revision"): 1688 if not hasattr(self, "_latest_revision"):
1574 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) 1689 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
1575 1690
1576 revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
1577 key = self.generate_revision_key(ud, d, name) 1691 key = self.generate_revision_key(ud, d, name)
1578 try: 1692
1579 return revs[key] 1693 rev = _revisions_cache.get_rev(key)
1580 except KeyError: 1694 if rev is None:
1581 revs[key] = rev = self._latest_revision(ud, d, name) 1695 rev = self._latest_revision(ud, d, name)
1582 return rev 1696 _revisions_cache.set_rev(key, rev)
1697 return rev
1583 1698
1584 def sortable_revision(self, ud, d, name): 1699 def sortable_revision(self, ud, d, name):
1585 latest_rev = self._build_revision(ud, d, name) 1700 latest_rev = self._build_revision(ud, d, name)
@@ -1611,12 +1726,61 @@ class FetchMethod(object):
1611 """ 1726 """
1612 return [] 1727 return []
1613 1728
1729
1730class DummyUnpackTracer(object):
1731 """
1732 Abstract API definition for a class that traces unpacked source files back
1733 to their respective upstream SRC_URI entries, for software composition
1734 analysis, license compliance and detailed SBOM generation purposes.
1735 User may load their own unpack tracer class (instead of the dummy
1736 one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
1737 """
1738 def start(self, unpackdir, urldata_dict, d):
1739 """
1740 Start tracing the core Fetch.unpack process, using an index to map
1741 unpacked files to each SRC_URI entry.
1742 This method is called by Fetch.unpack and it may receive nested calls by
1743 gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
1744 URLs and by recursively calling Fetch.unpack from new (nested) Fetch
1745 instances.
1746 """
1747 return
1748 def start_url(self, url):
1749 """Start tracing url unpack process.
1750 This method is called by Fetch.unpack before the fetcher-specific unpack
1751 method starts, and it may receive nested calls by gitsm and npmsw
1752 fetchers.
1753 """
1754 return
1755 def unpack(self, unpack_type, destdir):
1756 """
1757 Set unpack_type and destdir for current url.
1758 This method is called by the fetcher-specific unpack method after url
1759 tracing started.
1760 """
1761 return
1762 def finish_url(self, url):
1763 """Finish tracing url unpack process and update the file index.
1764 This method is called by Fetch.unpack after the fetcher-specific unpack
1765 method finished its job, and it may receive nested calls by gitsm
1766 and npmsw fetchers.
1767 """
1768 return
1769 def complete(self):
1770 """
1771 Finish tracing the Fetch.unpack process, and check if all nested
1772 Fecth.unpack calls (if any) have been completed; if so, save collected
1773 metadata.
1774 """
1775 return
1776
1777
1614class Fetch(object): 1778class Fetch(object):
1615 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): 1779 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
1616 if localonly and cache: 1780 if localonly and cache:
1617 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") 1781 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
1618 1782
1619 if len(urls) == 0: 1783 if not urls:
1620 urls = d.getVar("SRC_URI").split() 1784 urls = d.getVar("SRC_URI").split()
1621 self.urls = urls 1785 self.urls = urls
1622 self.d = d 1786 self.d = d
@@ -1631,10 +1795,30 @@ class Fetch(object):
1631 if key in urldata_cache: 1795 if key in urldata_cache:
1632 self.ud = urldata_cache[key] 1796 self.ud = urldata_cache[key]
1633 1797
1798 # the unpack_tracer object needs to be made available to possible nested
1799 # Fetch instances (when those are created by gitsm and npmsw fetchers)
1800 # so we set it as a global variable
1801 global unpack_tracer
1802 try:
1803 unpack_tracer
1804 except NameError:
1805 class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
1806 if class_path:
1807 # use user-defined unpack tracer class
1808 import importlib
1809 module_name, _, class_name = class_path.rpartition(".")
1810 module = importlib.import_module(module_name)
1811 class_ = getattr(module, class_name)
1812 unpack_tracer = class_()
1813 else:
1814 # fall back to the dummy/abstract class
1815 unpack_tracer = DummyUnpackTracer()
1816
1634 for url in urls: 1817 for url in urls:
1635 if url not in self.ud: 1818 if url not in self.ud:
1636 try: 1819 try:
1637 self.ud[url] = FetchData(url, d, localonly) 1820 self.ud[url] = FetchData(url, d, localonly)
1821 self.ud[url].unpack_tracer = unpack_tracer
1638 except NonLocalMethod: 1822 except NonLocalMethod:
1639 if localonly: 1823 if localonly:
1640 self.ud[url] = None 1824 self.ud[url] = None
@@ -1648,7 +1832,7 @@ class Fetch(object):
1648 self.ud[url] = FetchData(url, self.d) 1832 self.ud[url] = FetchData(url, self.d)
1649 1833
1650 self.ud[url].setup_localpath(self.d) 1834 self.ud[url].setup_localpath(self.d)
1651 return self.d.expand(self.ud[url].localpath) 1835 return self.ud[url].localpath
1652 1836
1653 def localpaths(self): 1837 def localpaths(self):
1654 """ 1838 """
@@ -1673,6 +1857,7 @@ class Fetch(object):
1673 network = self.d.getVar("BB_NO_NETWORK") 1857 network = self.d.getVar("BB_NO_NETWORK")
1674 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) 1858 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
1675 1859
1860 checksum_missing_messages = []
1676 for u in urls: 1861 for u in urls:
1677 ud = self.ud[u] 1862 ud = self.ud[u]
1678 ud.setup_localpath(self.d) 1863 ud.setup_localpath(self.d)
@@ -1684,7 +1869,6 @@ class Fetch(object):
1684 1869
1685 try: 1870 try:
1686 self.d.setVar("BB_NO_NETWORK", network) 1871 self.d.setVar("BB_NO_NETWORK", network)
1687
1688 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): 1872 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
1689 done = True 1873 done = True
1690 elif m.try_premirror(ud, self.d): 1874 elif m.try_premirror(ud, self.d):
@@ -1701,23 +1885,28 @@ class Fetch(object):
1701 logger.debug(str(e)) 1885 logger.debug(str(e))
1702 done = False 1886 done = False
1703 1887
1888 d = self.d
1704 if premirroronly: 1889 if premirroronly:
1705 self.d.setVar("BB_NO_NETWORK", "1") 1890 # Only disable the network in a copy
1891 d = bb.data.createCopy(self.d)
1892 d.setVar("BB_NO_NETWORK", "1")
1706 1893
1707 firsterr = None 1894 firsterr = None
1708 verified_stamp = m.verify_donestamp(ud, self.d) 1895 verified_stamp = False
1709 if not done and (not verified_stamp or m.need_update(ud, self.d)): 1896 if done:
1897 verified_stamp = m.verify_donestamp(ud, d)
1898 if not done and (not verified_stamp or m.need_update(ud, d)):
1710 try: 1899 try:
1711 if not trusted_network(self.d, ud.url): 1900 if not trusted_network(d, ud.url):
1712 raise UntrustedUrl(ud.url) 1901 raise UntrustedUrl(ud.url)
1713 logger.debug("Trying Upstream") 1902 logger.debug("Trying Upstream")
1714 m.download(ud, self.d) 1903 m.download(ud, d)
1715 if hasattr(m, "build_mirror_data"): 1904 if hasattr(m, "build_mirror_data"):
1716 m.build_mirror_data(ud, self.d) 1905 m.build_mirror_data(ud, d)
1717 done = True 1906 done = True
1718 # early checksum verify, so that if checksum mismatched, 1907 # early checksum verify, so that if checksum mismatched,
1719 # fetcher still have chance to fetch from mirror 1908 # fetcher still have chance to fetch from mirror
1720 m.update_donestamp(ud, self.d) 1909 m.update_donestamp(ud, d)
1721 1910
1722 except bb.fetch2.NetworkAccess: 1911 except bb.fetch2.NetworkAccess:
1723 raise 1912 raise
@@ -1735,18 +1924,18 @@ class Fetch(object):
1735 logger.debug(str(e)) 1924 logger.debug(str(e))
1736 firsterr = e 1925 firsterr = e
1737 # Remove any incomplete fetch 1926 # Remove any incomplete fetch
1738 if not verified_stamp: 1927 if not verified_stamp and m.cleanup_upon_failure():
1739 m.clean(ud, self.d) 1928 m.clean(ud, d)
1740 logger.debug("Trying MIRRORS") 1929 logger.debug("Trying MIRRORS")
1741 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1930 mirrors = mirror_from_string(d.getVar('MIRRORS'))
1742 done = m.try_mirrors(self, ud, self.d, mirrors) 1931 done = m.try_mirrors(self, ud, d, mirrors)
1743 1932
1744 if not done or not m.done(ud, self.d): 1933 if not done or not m.done(ud, d):
1745 if firsterr: 1934 if firsterr:
1746 logger.error(str(firsterr)) 1935 logger.error(str(firsterr))
1747 raise FetchError("Unable to fetch URL from any source.", u) 1936 raise FetchError("Unable to fetch URL from any source.", u)
1748 1937
1749 m.update_donestamp(ud, self.d) 1938 m.update_donestamp(ud, d)
1750 1939
1751 except IOError as e: 1940 except IOError as e:
1752 if e.errno in [errno.ESTALE]: 1941 if e.errno in [errno.ESTALE]:
@@ -1754,17 +1943,28 @@ class Fetch(object):
1754 raise ChecksumError("Stale Error Detected") 1943 raise ChecksumError("Stale Error Detected")
1755 1944
1756 except BBFetchException as e: 1945 except BBFetchException as e:
1757 if isinstance(e, ChecksumError): 1946 if isinstance(e, NoChecksumError):
1947 (message, _) = e.args
1948 checksum_missing_messages.append(message)
1949 continue
1950 elif isinstance(e, ChecksumError):
1758 logger.error("Checksum failure fetching %s" % u) 1951 logger.error("Checksum failure fetching %s" % u)
1759 raise 1952 raise
1760 1953
1761 finally: 1954 finally:
1762 if ud.lockfile: 1955 if ud.lockfile:
1763 bb.utils.unlockfile(lf) 1956 bb.utils.unlockfile(lf)
1957 if checksum_missing_messages:
1958 logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
1959 raise BBFetchException("There was some missing checksums in the recipe")
1764 1960
1765 def checkstatus(self, urls=None): 1961 def checkstatus(self, urls=None):
1766 """ 1962 """
1767 Check all urls exist upstream 1963 Check all URLs exist upstream.
1964
1965 Returns None if the URLs exist, raises FetchError if the check wasn't
1966 successful but there wasn't an error (such as file not found), and
1967 raises other exceptions in error cases.
1768 """ 1968 """
1769 1969
1770 if not urls: 1970 if not urls:
@@ -1787,7 +1987,7 @@ class Fetch(object):
1787 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1987 ret = m.try_mirrors(self, ud, self.d, mirrors, True)
1788 1988
1789 if not ret: 1989 if not ret:
1790 raise FetchError("URL %s doesn't work" % u, u) 1990 raise FetchError("URL doesn't work", u)
1791 1991
1792 def unpack(self, root, urls=None): 1992 def unpack(self, root, urls=None):
1793 """ 1993 """
@@ -1797,6 +1997,8 @@ class Fetch(object):
1797 if not urls: 1997 if not urls:
1798 urls = self.urls 1998 urls = self.urls
1799 1999
2000 unpack_tracer.start(root, self.ud, self.d)
2001
1800 for u in urls: 2002 for u in urls:
1801 ud = self.ud[u] 2003 ud = self.ud[u]
1802 ud.setup_localpath(self.d) 2004 ud.setup_localpath(self.d)
@@ -1804,11 +2006,15 @@ class Fetch(object):
1804 if ud.lockfile: 2006 if ud.lockfile:
1805 lf = bb.utils.lockfile(ud.lockfile) 2007 lf = bb.utils.lockfile(ud.lockfile)
1806 2008
2009 unpack_tracer.start_url(u)
1807 ud.method.unpack(ud, root, self.d) 2010 ud.method.unpack(ud, root, self.d)
2011 unpack_tracer.finish_url(u)
1808 2012
1809 if ud.lockfile: 2013 if ud.lockfile:
1810 bb.utils.unlockfile(lf) 2014 bb.utils.unlockfile(lf)
1811 2015
2016 unpack_tracer.complete()
2017
1812 def clean(self, urls=None): 2018 def clean(self, urls=None):
1813 """ 2019 """
1814 Clean files that the fetcher gets or places 2020 Clean files that the fetcher gets or places
@@ -1908,6 +2114,10 @@ from . import repo
1908from . import clearcase 2114from . import clearcase
1909from . import npm 2115from . import npm
1910from . import npmsw 2116from . import npmsw
2117from . import az
2118from . import crate
2119from . import gcp
2120from . import gomod
1911 2121
1912methods.append(local.Local()) 2122methods.append(local.Local())
1913methods.append(wget.Wget()) 2123methods.append(wget.Wget())
@@ -1927,3 +2137,8 @@ methods.append(repo.Repo())
1927methods.append(clearcase.ClearCase()) 2137methods.append(clearcase.ClearCase())
1928methods.append(npm.Npm()) 2138methods.append(npm.Npm())
1929methods.append(npmsw.NpmShrinkWrap()) 2139methods.append(npmsw.NpmShrinkWrap())
2140methods.append(az.Az())
2141methods.append(crate.Crate())
2142methods.append(gcp.GCP())
2143methods.append(gomod.GoMod())
2144methods.append(gomod.GoModGit())
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py
new file mode 100644
index 0000000000..1d3664f213
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/az.py
@@ -0,0 +1,98 @@
1"""
2BitBake 'Fetch' Azure Storage implementation
3
4"""
5
6# Copyright (C) 2021 Alejandro Hernandez Samaniego
7#
8# Based on bb.fetch2.wget:
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import shlex
16import os
17import bb
18from bb.fetch2 import FetchError
19from bb.fetch2 import logger
20from bb.fetch2.wget import Wget
21
22
23class Az(Wget):
24
25 def supports(self, ud, d):
26 """
27 Check to see if a given url can be fetched from Azure Storage
28 """
29 return ud.type in ['az']
30
31
32 def checkstatus(self, fetch, ud, d, try_again=True):
33
34 # checkstatus discards parameters either way, we need to do this before adding the SAS
35 ud.url = ud.url.replace('az://','https://').split(';')[0]
36
37 az_sas = d.getVar('AZ_SAS')
38 if az_sas and az_sas not in ud.url:
39 if not az_sas.startswith('?'):
40 raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
41 ud.url += az_sas
42
43 return Wget.checkstatus(self, fetch, ud, d, try_again)
44
45 # Override download method, include retries
46 def download(self, ud, d, retries=3):
47 """Fetch urls"""
48
49 # If were reaching the account transaction limit we might be refused a connection,
50 # retrying allows us to avoid false negatives since the limit changes over time
51 fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
52
53 # We need to provide a localpath to avoid wget using the SAS
54 # ud.localfile either has the downloadfilename or ud.path
55 localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
56 bb.utils.mkdirhier(os.path.dirname(localpath))
57 fetchcmd += " -O %s" % shlex.quote(localpath)
58
59
60 if ud.user and ud.pswd:
61 fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
62
63 # Check if a Shared Access Signature was given and use it
64 az_sas = d.getVar('AZ_SAS')
65
66 if az_sas:
67 if not az_sas.startswith('?'):
68 raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
69 azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
70 else:
71 azuri = '%s%s%s' % ('https://', ud.host, ud.path)
72
73 dldir = d.getVar("DL_DIR")
74 if os.path.exists(ud.localpath):
75 # file exists, but we didnt complete it.. trying again.
76 fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
77 else:
78 fetchcmd += " -P %s '%s'" % (dldir, azuri)
79
80 try:
81 self._runwget(ud, d, fetchcmd, False)
82 except FetchError as e:
83 # Azure fails on handshake sometimes when using wget after some stress, producing a
84 # FetchError from the fetcher, if the artifact exists retyring should succeed
85 if 'Unable to establish SSL connection' in str(e):
86 logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
87 self.download(ud, d, retries -1)
88
89 # Sanity check since wget can pretend it succeed when it didn't
90 # Also, this used to happen if sourceforge sent us to the mirror page
91 if not os.path.exists(ud.localpath):
92 raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
93
94 if os.path.getsize(ud.localpath) == 0:
95 os.remove(ud.localpath)
96 raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
97
98 return True
diff --git a/bitbake/lib/bb/fetch2/clearcase.py b/bitbake/lib/bb/fetch2/clearcase.py
index 1a9c863769..17500daf95 100644
--- a/bitbake/lib/bb/fetch2/clearcase.py
+++ b/bitbake/lib/bb/fetch2/clearcase.py
@@ -108,7 +108,7 @@ class ClearCase(FetchMethod):
108 ud.module.replace("/", "."), 108 ud.module.replace("/", "."),
109 ud.label.replace("/", ".")) 109 ud.label.replace("/", "."))
110 110
111 ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True)) 111 ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME"))
112 ud.csname = "%s-config-spec" % (ud.identifier) 112 ud.csname = "%s-config-spec" % (ud.identifier)
113 ud.ccasedir = os.path.join(d.getVar("DL_DIR"), ud.type) 113 ud.ccasedir = os.path.join(d.getVar("DL_DIR"), ud.type)
114 ud.viewdir = os.path.join(ud.ccasedir, ud.viewname) 114 ud.viewdir = os.path.join(ud.ccasedir, ud.viewname)
@@ -130,8 +130,6 @@ class ClearCase(FetchMethod):
130 self.debug("configspecfile = %s" % ud.configspecfile) 130 self.debug("configspecfile = %s" % ud.configspecfile)
131 self.debug("localfile = %s" % ud.localfile) 131 self.debug("localfile = %s" % ud.localfile)
132 132
133 ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
134
135 def _build_ccase_command(self, ud, command): 133 def _build_ccase_command(self, ud, command):
136 """ 134 """
137 Build up a commandline based on ud 135 Build up a commandline based on ud
@@ -196,7 +194,7 @@ class ClearCase(FetchMethod):
196 194
197 def need_update(self, ud, d): 195 def need_update(self, ud, d):
198 if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec): 196 if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec):
199 ud.identifier += "-%s" % d.getVar("DATETIME",d, True) 197 ud.identifier += "-%s" % d.getVar("DATETIME")
200 return True 198 return True
201 if os.path.exists(ud.localpath): 199 if os.path.exists(ud.localpath):
202 return False 200 return False
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
new file mode 100644
index 0000000000..e611736f06
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -0,0 +1,150 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementation for crates.io
5"""
6
7# Copyright (C) 2016 Doug Goldstein
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11# Based on functions from the base bb module, Copyright 2003 Holger Schurig
12
13import hashlib
14import json
15import os
16import subprocess
17import bb
18from bb.fetch2 import logger, subprocess_setup, UnpackError
19from bb.fetch2.wget import Wget
20
21
22class Crate(Wget):
23
24 """Class to fetch crates via wget"""
25
26 def _cargo_bitbake_path(self, rootdir):
27 return os.path.join(rootdir, "cargo_home", "bitbake")
28
29 def supports(self, ud, d):
30 """
31 Check to see if a given url is for this fetcher
32 """
33 return ud.type in ['crate']
34
35 def recommends_checksum(self, urldata):
36 return True
37
38 def urldata_init(self, ud, d):
39 """
40 Sets up to download the respective crate from crates.io
41 """
42
43 if ud.type == 'crate':
44 self._crate_urldata_init(ud, d)
45
46 super(Crate, self).urldata_init(ud, d)
47
48 def _crate_urldata_init(self, ud, d):
49 """
50 Sets up the download for a crate
51 """
52
53 # URL syntax is: crate://NAME/VERSION
54 # break the URL apart by /
55 parts = ud.url.split('/')
56 if len(parts) < 5:
57 raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
58
59 # version is expected to be the last token
60 # but ignore possible url parameters which will be used
61 # by the top fetcher class
62 version = parts[-1].split(";")[0]
63 # second to last field is name
64 name = parts[-2]
65 # host (this is to allow custom crate registries to be specified
66 host = '/'.join(parts[2:-2])
67
68 # if using upstream just fix it up nicely
69 if host == 'crates.io':
70 host = 'crates.io/api/v1/crates'
71
72 ud.url = "https://%s/%s/%s/download" % (host, name, version)
73 ud.versionsurl = "https://%s/%s/versions" % (host, name)
74 ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
75 if 'name' not in ud.parm:
76 ud.parm['name'] = '%s-%s' % (name, version)
77
78 logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
79
80 def unpack(self, ud, rootdir, d):
81 """
82 Uses the crate to build the necessary paths for cargo to utilize it
83 """
84 if ud.type == 'crate':
85 return self._crate_unpack(ud, rootdir, d)
86 else:
87 super(Crate, self).unpack(ud, rootdir, d)
88
89 def _crate_unpack(self, ud, rootdir, d):
90 """
91 Unpacks a crate
92 """
93 thefile = ud.localpath
94
95 # possible metadata we need to write out
96 metadata = {}
97
98 # change to the rootdir to unpack but save the old working dir
99 save_cwd = os.getcwd()
100 os.chdir(rootdir)
101
102 bp = d.getVar('BP')
103 if bp == ud.parm.get('name'):
104 cmd = "tar -xz --no-same-owner -f %s" % thefile
105 ud.unpack_tracer.unpack("crate-extract", rootdir)
106 else:
107 cargo_bitbake = self._cargo_bitbake_path(rootdir)
108 ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
109
110 cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
111
112 # ensure we've got these paths made
113 bb.utils.mkdirhier(cargo_bitbake)
114
115 # generate metadata necessary
116 with open(thefile, 'rb') as f:
117 # get the SHA256 of the original tarball
118 tarhash = hashlib.sha256(f.read()).hexdigest()
119
120 metadata['files'] = {}
121 metadata['package'] = tarhash
122
123 path = d.getVar('PATH')
124 if path:
125 cmd = "PATH=\"%s\" %s" % (path, cmd)
126 bb.note("Unpacking %s to %s/" % (thefile, os.getcwd()))
127
128 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
129
130 os.chdir(save_cwd)
131
132 if ret != 0:
133 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
134
135 # if we have metadata to write out..
136 if len(metadata) > 0:
137 cratepath = os.path.splitext(os.path.basename(thefile))[0]
138 bbpath = self._cargo_bitbake_path(rootdir)
139 mdfile = '.cargo-checksum.json'
140 mdpath = os.path.join(bbpath, cratepath, mdfile)
141 with open(mdpath, "w") as f:
142 json.dump(metadata, f)
143
144 def latest_versionstring(self, ud, d):
145 from functools import cmp_to_key
146 json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
147 versions = [(0, i["num"], "") for i in json_data["versions"]]
148 versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
149
150 return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
new file mode 100644
index 0000000000..86546d40bf
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -0,0 +1,102 @@
1"""
2BitBake 'Fetch' implementation for Google Cloup Platform Storage.
3
4Class for fetching files from Google Cloud Storage using the
5Google Cloud Storage Python Client. The GCS Python Client must
6be correctly installed, configured and authenticated prior to use.
7Additionally, gsutil must also be installed.
8
9"""
10
11# Copyright (C) 2023, Snap Inc.
12#
13# Based in part on bb.fetch2.s3:
14# Copyright (C) 2017 Andre McCurdy
15#
16# SPDX-License-Identifier: GPL-2.0-only
17#
18# Based on functions from the base bb module, Copyright 2003 Holger Schurig
19
20import os
21import bb
22import urllib.parse, urllib.error
23from bb.fetch2 import FetchMethod
24from bb.fetch2 import FetchError
25from bb.fetch2 import logger
26
27class GCP(FetchMethod):
28 """
29 Class to fetch urls via GCP's Python API.
30 """
31 def __init__(self):
32 self.gcp_client = None
33
34 def supports(self, ud, d):
35 """
36 Check to see if a given url can be fetched with GCP.
37 """
38 return ud.type in ['gs']
39
40 def recommends_checksum(self, urldata):
41 return True
42
43 def urldata_init(self, ud, d):
44 if 'downloadfilename' in ud.parm:
45 ud.basename = ud.parm['downloadfilename']
46 else:
47 ud.basename = os.path.basename(ud.path)
48
49 ud.localfile = ud.basename
50
51 def get_gcp_client(self):
52 from google.cloud import storage
53 self.gcp_client = storage.Client(project=None)
54
55 def download(self, ud, d):
56 """
57 Fetch urls using the GCP API.
58 Assumes localpath was called first.
59 """
60 from google.api_core.exceptions import NotFound
61 logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
62 if self.gcp_client is None:
63 self.get_gcp_client()
64
65 bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}")
66
67 # Path sometimes has leading slash, so strip it
68 path = ud.path.lstrip("/")
69 blob = self.gcp_client.bucket(ud.host).blob(path)
70 try:
71 blob.download_to_filename(ud.localpath)
72 except NotFound:
73 raise FetchError("The GCP API threw a NotFound exception")
74
75 # Additional sanity checks copied from the wget class (although there
76 # are no known issues which mean these are required, treat the GCP API
77 # tool with a little healthy suspicion).
78 if not os.path.exists(ud.localpath):
79 raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
80
81 if os.path.getsize(ud.localpath) == 0:
82 os.remove(ud.localpath)
83 raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
84
85 return True
86
87 def checkstatus(self, fetch, ud, d):
88 """
89 Check the status of a URL.
90 """
91 logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
92 if self.gcp_client is None:
93 self.get_gcp_client()
94
95 bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}")
96
97 # Path sometimes has leading slash, so strip it
98 path = ud.path.lstrip("/")
99 if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
100 raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
101 else:
102 return True
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e3ba80a3f5..14ec45a3f6 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -9,15 +9,6 @@ Supported SRC_URI options are:
9- branch 9- branch
10 The git branch to retrieve from. The default is "master" 10 The git branch to retrieve from. The default is "master"
11 11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag 12- tag
22 The git tag to retrieve. The default is "master" 13 The git tag to retrieve. The default is "master"
23 14
@@ -44,13 +35,27 @@ Supported SRC_URI options are:
44 35
45- nobranch 36- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe 37 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch. 38 referring to commit which is valid in any namespace (branch, tag, ...)
39 instead of branch.
48 The default is "0", set nobranch=1 if needed. 40 The default is "0", set nobranch=1 if needed.
49 41
42- subpath
43 Limit the checkout to a specific subpath of the tree.
44 By default, checkout the whole tree, set subpath=<path> if needed
45
46- destsuffix
47 The name of the path in which to place the checkout.
48 By default, the path is git/, set destsuffix=<suffix> if needed
49
50- usehead 50- usehead
51 For local git:// urls to use the current branch HEAD as the revision for use with 51 For local git:// urls to use the current branch HEAD as the revision for use with
52 AUTOREV. Implies nobranch. 52 AUTOREV. Implies nobranch.
53 53
54- lfs
55 Enable the checkout to use LFS for large files. This will download all LFS files
56 in the download step, as the unpack step does not have network access.
57 The default is "1", set lfs=0 to skip.
58
54""" 59"""
55 60
56# Copyright (C) 2005 Richard Purdie 61# Copyright (C) 2005 Richard Purdie
@@ -64,15 +69,22 @@ import fnmatch
64import os 69import os
65import re 70import re
66import shlex 71import shlex
72import shutil
67import subprocess 73import subprocess
68import tempfile 74import tempfile
75import urllib
69import bb 76import bb
70import bb.progress 77import bb.progress
78from contextlib import contextmanager
71from bb.fetch2 import FetchMethod 79from bb.fetch2 import FetchMethod
72from bb.fetch2 import runfetchcmd 80from bb.fetch2 import runfetchcmd
73from bb.fetch2 import logger 81from bb.fetch2 import logger
82from bb.fetch2 import trusted_network
74 83
75 84
85sha1_re = re.compile(r'^[0-9a-f]{40}$')
86slash_re = re.compile(r"/+")
87
76class GitProgressHandler(bb.progress.LineFilterProgressHandler): 88class GitProgressHandler(bb.progress.LineFilterProgressHandler):
77 """Extract progress information from git output""" 89 """Extract progress information from git output"""
78 def __init__(self, d): 90 def __init__(self, d):
@@ -130,6 +142,9 @@ class Git(FetchMethod):
130 def supports_checksum(self, urldata): 142 def supports_checksum(self, urldata):
131 return False 143 return False
132 144
145 def cleanup_upon_failure(self):
146 return False
147
133 def urldata_init(self, ud, d): 148 def urldata_init(self, ud, d):
134 """ 149 """
135 init git specific variable within url data 150 init git specific variable within url data
@@ -141,6 +156,11 @@ class Git(FetchMethod):
141 ud.proto = 'file' 156 ud.proto = 'file'
142 else: 157 else:
143 ud.proto = "git" 158 ud.proto = "git"
159 if ud.host == "github.com" and ud.proto == "git":
160 # github stopped supporting git protocol
161 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
162 ud.proto = "https"
163 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
144 164
145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): 165 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) 166 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -162,18 +182,25 @@ class Git(FetchMethod):
162 ud.bareclone = ud.parm.get("bareclone","0") == "1" 182 ud.bareclone = ud.parm.get("bareclone","0") == "1"
163 if ud.bareclone: 183 if ud.bareclone:
164 ud.nocheckout = 1 184 ud.nocheckout = 1
165
166 ud.unresolvedrev = {}
167 branches = ud.parm.get("branch", "master").split(',')
168 if len(branches) != len(ud.names):
169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
170 185
171 ud.cloneflags = "-s -n" 186 ud.unresolvedrev = ""
187 ud.branch = ud.parm.get("branch", "")
188 if not ud.branch and not ud.nobranch:
189 raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
190
191 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
192
193 ud.cloneflags = "-n"
194 if not ud.noshared:
195 ud.cloneflags += " -s"
172 if ud.bareclone: 196 if ud.bareclone:
173 ud.cloneflags += " --mirror" 197 ud.cloneflags += " --mirror"
174 198
199 ud.shallow_skip_fast = False
175 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" 200 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
176 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() 201 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
202 if 'tag' in ud.parm:
203 ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag'])
177 204
178 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") 205 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
179 if depth_default is not None: 206 if depth_default is not None:
@@ -190,32 +217,27 @@ class Git(FetchMethod):
190 217
191 revs_default = d.getVar("BB_GIT_SHALLOW_REVS") 218 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
192 ud.shallow_revs = [] 219 ud.shallow_revs = []
193 ud.branches = {} 220
194 for pos, name in enumerate(ud.names): 221 ud.unresolvedrev = ud.branch
195 branch = branches[pos] 222
196 ud.branches[name] = branch 223 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
197 ud.unresolvedrev[name] = branch 224 if shallow_depth is not None:
198 225 try:
199 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) 226 shallow_depth = int(shallow_depth or 0)
200 if shallow_depth is not None: 227 except ValueError:
201 try: 228 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
202 shallow_depth = int(shallow_depth or 0) 229 else:
203 except ValueError: 230 if shallow_depth < 0:
204 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 231 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
205 else: 232 ud.shallow_depths[ud.name] = shallow_depth
206 if shallow_depth < 0: 233
207 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 234 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
208 ud.shallow_depths[name] = shallow_depth 235 if revs is not None:
209 236 ud.shallow_revs.extend(revs.split())
210 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) 237 elif revs_default is not None:
211 if revs is not None: 238 ud.shallow_revs.extend(revs_default.split())
212 ud.shallow_revs.extend(revs.split()) 239
213 elif revs_default is not None: 240 if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
214 ud.shallow_revs.extend(revs_default.split())
215
216 if (ud.shallow and
217 not ud.shallow_revs and
218 all(ud.shallow_depths[n] == 0 for n in ud.names)):
219 # Shallow disabled for this URL 241 # Shallow disabled for this URL
220 ud.shallow = False 242 ud.shallow = False
221 243
@@ -224,10 +246,9 @@ class Git(FetchMethod):
224 # rev of this repository. This will get resolved into a revision 246 # rev of this repository. This will get resolved into a revision
225 # later. If an actual revision happens to have also been provided 247 # later. If an actual revision happens to have also been provided
226 # then this setting will be overridden. 248 # then this setting will be overridden.
227 for name in ud.names: 249 ud.unresolvedrev = 'HEAD'
228 ud.unresolvedrev[name] = 'HEAD'
229 250
230 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" 251 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
231 252
232 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" 253 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
233 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable 254 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -235,24 +256,22 @@ class Git(FetchMethod):
235 256
236 ud.setup_revisions(d) 257 ud.setup_revisions(d)
237 258
238 for name in ud.names: 259 # Ensure any revision that doesn't look like a SHA-1 is translated into one
239 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one 260 if not sha1_re.match(ud.revision or ''):
240 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): 261 if ud.revision:
241 if ud.revisions[name]: 262 ud.unresolvedrev = ud.revision
242 ud.unresolvedrev[name] = ud.revisions[name] 263 ud.revision = self.latest_revision(ud, d, ud.name)
243 ud.revisions[name] = self.latest_revision(ud, d, name)
244 264
245 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) 265 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
246 if gitsrcname.startswith('.'): 266 if gitsrcname.startswith('.'):
247 gitsrcname = gitsrcname[1:] 267 gitsrcname = gitsrcname[1:]
248 268
249 # for rebaseable git repo, it is necessary to keep mirror tar ball 269 # For a rebaseable git repo, it is necessary to keep a mirror tar ball
250 # per revision, so that even the revision disappears from the 270 # per revision, so that even if the revision disappears from the
251 # upstream repo in the future, the mirror will remain intact and still 271 # upstream repo in the future, the mirror will remain intact and still
252 # contains the revision 272 # contain the revision
253 if ud.rebaseable: 273 if ud.rebaseable:
254 for name in ud.names: 274 gitsrcname = gitsrcname + '_' + ud.revision
255 gitsrcname = gitsrcname + '_' + ud.revisions[name]
256 275
257 dl_dir = d.getVar("DL_DIR") 276 dl_dir = d.getVar("DL_DIR")
258 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") 277 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -270,15 +289,14 @@ class Git(FetchMethod):
270 if ud.shallow_revs: 289 if ud.shallow_revs:
271 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) 290 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
272 291
273 for name, revision in sorted(ud.revisions.items()): 292 tarballname = "%s_%s" % (tarballname, ud.revision[:7])
274 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 293 depth = ud.shallow_depths[ud.name]
275 depth = ud.shallow_depths[name] 294 if depth:
276 if depth: 295 tarballname = "%s-%s" % (tarballname, depth)
277 tarballname = "%s-%s" % (tarballname, depth)
278 296
279 shallow_refs = [] 297 shallow_refs = []
280 if not ud.nobranch: 298 if not ud.nobranch:
281 shallow_refs.extend(ud.branches.values()) 299 shallow_refs.append(ud.branch)
282 if ud.shallow_extra_refs: 300 if ud.shallow_extra_refs:
283 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) 301 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
284 if shallow_refs: 302 if shallow_refs:
@@ -293,16 +311,29 @@ class Git(FetchMethod):
293 return ud.clonedir 311 return ud.clonedir
294 312
295 def need_update(self, ud, d): 313 def need_update(self, ud, d):
296 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) 314 return self.clonedir_need_update(ud, d) \
315 or self.shallow_tarball_need_update(ud) \
316 or self.tarball_need_update(ud) \
317 or self.lfs_need_update(ud, d)
297 318
298 def clonedir_need_update(self, ud, d): 319 def clonedir_need_update(self, ud, d):
299 if not os.path.exists(ud.clonedir): 320 if not os.path.exists(ud.clonedir):
300 return True 321 return True
301 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): 322 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
302 return True 323 return True
303 for name in ud.names: 324 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
304 if not self._contains_ref(ud, d, name, ud.clonedir): 325 return True
305 return True 326 return False
327
328 def lfs_need_update(self, ud, d):
329 if not self._need_lfs(ud):
330 return False
331
332 if self.clonedir_need_update(ud, d):
333 return True
334
335 if not self._lfs_objects_downloaded(ud, d, ud.clonedir):
336 return True
306 return False 337 return False
307 338
308 def clonedir_need_shallow_revs(self, ud, d): 339 def clonedir_need_shallow_revs(self, ud, d):
@@ -319,11 +350,28 @@ class Git(FetchMethod):
319 def tarball_need_update(self, ud): 350 def tarball_need_update(self, ud):
320 return ud.write_tarballs and not os.path.exists(ud.fullmirror) 351 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
321 352
353 def update_mirror_links(self, ud, origud):
354 super().update_mirror_links(ud, origud)
355 # When using shallow mode, add a symlink to the original fullshallow
356 # path to ensure a valid symlink even in the `PREMIRRORS` case
357 if ud.shallow and not os.path.exists(origud.fullshallow):
358 self.ensure_symlink(ud.localpath, origud.fullshallow)
359
322 def try_premirror(self, ud, d): 360 def try_premirror(self, ud, d):
323 # If we don't do this, updating an existing checkout with only premirrors 361 # If we don't do this, updating an existing checkout with only premirrors
324 # is not possible 362 # is not possible
325 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): 363 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
326 return True 364 return True
365 # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
366 # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
367 # we need to try premirrors first as using upstream is destined to fail.
368 if not trusted_network(d, ud.url):
369 return True
370 # the following check is to ensure incremental fetch in downloads, this is
371 # because the premirror might be old and does not contain the new rev required,
372 # and this will cause a total removal and new clone. So if we can reach to
373 # network, we prefer upstream over premirror, though the premirror might contain
374 # the new rev.
327 if os.path.exists(ud.clonedir): 375 if os.path.exists(ud.clonedir):
328 return False 376 return False
329 return True 377 return True
@@ -337,21 +385,76 @@ class Git(FetchMethod):
337 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 385 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
338 ud.localpath = ud.fullshallow 386 ud.localpath = ud.fullshallow
339 return 387 return
340 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): 388 elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
341 bb.utils.mkdirhier(ud.clonedir) 389 if not os.path.exists(ud.clonedir):
342 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 390 bb.utils.mkdirhier(ud.clonedir)
343 391 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
392 else:
393 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
394 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
395 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
396 if 'mirror' in output:
397 runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
398 runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
399 fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd)
400 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
344 repourl = self._get_repo_url(ud) 401 repourl = self._get_repo_url(ud)
345 402
403 needs_clone = False
404 if os.path.exists(ud.clonedir):
405 # The directory may exist, but not be the top level of a bare git
406 # repository in which case it needs to be deleted and re-cloned.
407 try:
408 # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
409 output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
410 toplevel = output.rstrip()
411
412 if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
413 logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
414 needs_clone = True
415 except bb.fetch2.FetchError as e:
416 logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
417 needs_clone = True
418 except FileNotFoundError as e:
419 logger.warning("%s", e)
420 needs_clone = True
421
422 if needs_clone:
423 shutil.rmtree(ud.clonedir)
424 else:
425 needs_clone = True
426
346 # If the repo still doesn't exist, fallback to cloning it 427 # If the repo still doesn't exist, fallback to cloning it
347 if not os.path.exists(ud.clonedir): 428 if needs_clone:
348 # We do this since git will use a "-l" option automatically for local urls where possible 429 # We do this since git will use a "-l" option automatically for local urls where possible,
430 # but it doesn't work when git/objects is a symlink, only works when it is a directory.
349 if repourl.startswith("file://"): 431 if repourl.startswith("file://"):
350 repourl = repourl[7:] 432 repourl_path = repourl[7:]
433 objects = os.path.join(repourl_path, 'objects')
434 if os.path.isdir(objects) and not os.path.islink(objects):
435 repourl = repourl_path
351 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) 436 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
352 if ud.proto.lower() != 'file': 437 if ud.proto.lower() != 'file':
353 bb.fetch2.check_network_access(d, clone_cmd, ud.url) 438 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
354 progresshandler = GitProgressHandler(d) 439 progresshandler = GitProgressHandler(d)
440
441 # Try creating a fast initial shallow clone
442 # Enabling ud.shallow_skip_fast will skip this
443 # If the Git error "Server does not allow request for unadvertised object"
444 # occurs, shallow_skip_fast is enabled automatically.
445 # This may happen if the Git server does not allow the request
446 # or if the Git client has issues with this functionality.
447 if ud.shallow and not ud.shallow_skip_fast:
448 try:
449 self.clone_shallow_with_tarball(ud, d)
450 # When the shallow clone has succeeded, use the shallow tarball
451 ud.localpath = ud.fullshallow
452 return
453 except:
454 logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
455
456 # When skipping fast initial shallow or the fast inital shallow clone failed:
457 # Try again with an initial regular clone
355 runfetchcmd(clone_cmd, d, log=progresshandler) 458 runfetchcmd(clone_cmd, d, log=progresshandler)
356 459
357 # Update the checkout if needed 460 # Update the checkout if needed
@@ -361,7 +464,11 @@ class Git(FetchMethod):
361 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) 464 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
362 465
363 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) 466 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
364 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) 467
468 if ud.nobranch:
469 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
470 else:
471 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
365 if ud.proto.lower() != 'file': 472 if ud.proto.lower() != 'file':
366 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 473 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
367 progresshandler = GitProgressHandler(d) 474 progresshandler = GitProgressHandler(d)
@@ -375,138 +482,206 @@ class Git(FetchMethod):
375 if exc.errno != errno.ENOENT: 482 if exc.errno != errno.ENOENT:
376 raise 483 raise
377 484
378 for name in ud.names: 485 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
379 if not self._contains_ref(ud, d, name, ud.clonedir): 486 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
380 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
381 487
382 if ud.shallow and ud.write_shallow_tarballs: 488 if ud.shallow and ud.write_shallow_tarballs:
383 missing_rev = self.clonedir_need_shallow_revs(ud, d) 489 missing_rev = self.clonedir_need_shallow_revs(ud, d)
384 if missing_rev: 490 if missing_rev:
385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 491 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
386 492
387 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): 493 if self.lfs_need_update(ud, d):
388 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching 494 self.lfs_fetch(ud, d, ud.clonedir, ud.revision)
389 # of all LFS blobs needed at the the srcrev. 495
390 # 496 def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
391 # It would be nice to just do this inline here by running 'git-lfs fetch' 497 """Helper method for fetching Git LFS data"""
392 # on the bare clonedir, but that operation requires a working copy on some 498 try:
393 # releases of Git LFS. 499 if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision):
394 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 500 self._ensure_git_lfs(d, ud)
395 try: 501
396 # Do the checkout. This implicitly involves a Git LFS fetch. 502 # Using worktree with the revision because .lfsconfig may exists
397 self.unpack(ud, tmpdir, d) 503 worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
398 504 runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
399 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into 505 lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
400 # the bare clonedir. 506 runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
401 # 507 worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
402 # As this procedure is invoked repeatedly on incremental fetches as 508 runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
403 # a recipe's SRCREV is bumped throughout its lifetime, this will 509 except:
404 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs 510 logger.warning("Fetching LFS did not succeed.")
405 # corresponding to all the blobs reachable from the different revs 511
406 # fetched across time. 512 @contextmanager
407 # 513 def create_atomic(self, filename):
408 # Only do this if the unpack resulted in a .git/lfs directory being 514 """Create as a temp file and move atomically into position to avoid races"""
409 # created; this only happens if at least one blob needed to be 515 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
410 # downloaded. 516 try:
411 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): 517 yield tfile
412 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) 518 umask = os.umask(0o666)
413 finally: 519 os.umask(umask)
414 bb.utils.remove(tmpdir, recurse=True) 520 os.chmod(tfile, (0o666 & ~umask))
521 os.rename(tfile, filename)
522 finally:
523 os.close(fd)
415 524
416 def build_mirror_data(self, ud, d): 525 def build_mirror_data(self, ud, d):
417 if ud.shallow and ud.write_shallow_tarballs: 526 if ud.shallow and ud.write_shallow_tarballs:
418 if not os.path.exists(ud.fullshallow): 527 if not os.path.exists(ud.fullshallow):
419 if os.path.islink(ud.fullshallow): 528 if os.path.islink(ud.fullshallow):
420 os.unlink(ud.fullshallow) 529 os.unlink(ud.fullshallow)
421 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 530 self.clone_shallow_with_tarball(ud, d)
422 shallowclone = os.path.join(tempdir, 'git')
423 try:
424 self.clone_shallow_local(ud, shallowclone, d)
425
426 logger.info("Creating tarball of git repository")
427 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
428 runfetchcmd("touch %s.done" % ud.fullshallow, d)
429 finally:
430 bb.utils.remove(tempdir, recurse=True)
431 elif ud.write_tarballs and not os.path.exists(ud.fullmirror): 531 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
432 if os.path.islink(ud.fullmirror): 532 if os.path.islink(ud.fullmirror):
433 os.unlink(ud.fullmirror) 533 os.unlink(ud.fullmirror)
434 534
435 logger.info("Creating tarball of git repository") 535 logger.info("Creating tarball of git repository")
436 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 536 with self.create_atomic(ud.fullmirror) as tfile:
537 mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
538 quiet=True, workdir=ud.clonedir)
539 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
540 % (tfile, mtime), d, workdir=ud.clonedir)
437 runfetchcmd("touch %s.done" % ud.fullmirror, d) 541 runfetchcmd("touch %s.done" % ud.fullmirror, d)
438 542
543 def clone_shallow_with_tarball(self, ud, d):
544 ret = False
545 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
546 shallowclone = os.path.join(tempdir, 'git')
547 try:
548 try:
549 self.clone_shallow_local(ud, shallowclone, d)
550 except:
551 logger.warning("Fast shallow clone failed, try to skip fast mode now.")
552 bb.utils.remove(tempdir, recurse=True)
553 os.mkdir(tempdir)
554 ud.shallow_skip_fast = True
555 self.clone_shallow_local(ud, shallowclone, d)
556 logger.info("Creating tarball of git repository")
557 with self.create_atomic(ud.fullshallow) as tfile:
558 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
559 runfetchcmd("touch %s.done" % ud.fullshallow, d)
560 ret = True
561 finally:
562 bb.utils.remove(tempdir, recurse=True)
563
564 return ret
565
439 def clone_shallow_local(self, ud, dest, d): 566 def clone_shallow_local(self, ud, dest, d):
440 """Clone the repo and make it shallow. 567 """
568 Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
569 - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
570 - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
571 """
441 572
442 The upstream url of the new clone isn't set at this time, as it'll be 573 progresshandler = GitProgressHandler(d)
443 set correctly when unpacked.""" 574 repourl = self._get_repo_url(ud)
444 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) 575 bb.utils.mkdirhier(dest)
576 init_cmd = "%s init -q" % ud.basecmd
577 if ud.bareclone:
578 init_cmd += " --bare"
579 runfetchcmd(init_cmd, d, workdir=dest)
580 # Use repourl when creating a fast initial shallow clone
581 # Prefer already existing full bare clones if available
582 if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
583 remote = shlex.quote(repourl)
584 else:
585 remote = ud.clonedir
586 runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
445 587
446 to_parse, shallow_branches = [], [] 588 # Check the histories which should be excluded
447 for name in ud.names: 589 shallow_exclude = ''
448 revision = ud.revisions[name] 590 for revision in ud.shallow_revs:
449 depth = ud.shallow_depths[name] 591 shallow_exclude += " --shallow-exclude=%s" % revision
450 if depth:
451 to_parse.append('%s~%d^{}' % (revision, depth - 1))
452 592
453 # For nobranch, we need a ref, otherwise the commits will be 593 revision = ud.revision
454 # removed, and for non-nobranch, we truncate the branch to our 594 depth = ud.shallow_depths[ud.name]
455 # srcrev, to avoid keeping unnecessary history beyond that.
456 branch = ud.branches[name]
457 if ud.nobranch:
458 ref = "refs/shallow/%s" % name
459 elif ud.bareclone:
460 ref = "refs/heads/%s" % branch
461 else:
462 ref = "refs/remotes/origin/%s" % branch
463 595
464 shallow_branches.append(ref) 596 # The --depth and --shallow-exclude can't be used together
465 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 597 if depth and shallow_exclude:
598 raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
466 599
467 # Map srcrev+depths to revisions 600 # For nobranch, we need a ref, otherwise the commits will be
468 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) 601 # removed, and for non-nobranch, we truncate the branch to our
602 # srcrev, to avoid keeping unnecessary history beyond that.
603 branch = ud.branch
604 if ud.nobranch:
605 ref = "refs/shallow/%s" % ud.name
606 elif ud.bareclone:
607 ref = "refs/heads/%s" % branch
608 else:
609 ref = "refs/remotes/origin/%s" % branch
610
611 fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
612 if depth:
613 fetch_cmd += " --depth %s" % depth
614
615 if shallow_exclude:
616 fetch_cmd += shallow_exclude
469 617
470 # Resolve specified revisions 618 # Advertise the revision for lower version git such as 2.25.1:
471 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) 619 # error: Server does not allow request for unadvertised object.
472 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() 620 # The ud.clonedir is a local temporary dir, will be removed when
621 # fetch is done, so we can do anything on it.
622 adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
623 if ud.shallow_skip_fast:
624 runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
625
626 runfetchcmd(fetch_cmd, d, workdir=dest)
627 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
628 # Fetch Git LFS data
629 self.lfs_fetch(ud, d, dest, ud.revision)
473 630
474 # Apply extra ref wildcards 631 # Apply extra ref wildcards
475 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, 632 all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
476 d, workdir=dest).splitlines() 633 d, workdir=dest).splitlines()
634 all_refs = []
635 for line in all_refs_remote:
636 all_refs.append(line.split()[-1])
637 extra_refs = []
477 for r in ud.shallow_extra_refs: 638 for r in ud.shallow_extra_refs:
478 if not ud.bareclone: 639 if not ud.bareclone:
479 r = r.replace('refs/heads/', 'refs/remotes/origin/') 640 r = r.replace('refs/heads/', 'refs/remotes/origin/')
480 641
481 if '*' in r: 642 if '*' in r:
482 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) 643 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
483 shallow_branches.extend(matches) 644 extra_refs.extend(matches)
484 else: 645 else:
485 shallow_branches.append(r) 646 extra_refs.append(r)
647
648 for ref in extra_refs:
649 ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '')
650 runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
651 revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
652 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
486 653
487 # Make the repository shallow 654 # The url is local ud.clonedir, set it to upstream one
488 shallow_cmd = [self.make_shallow_path, '-s'] 655 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
489 for b in shallow_branches:
490 shallow_cmd.append('-r')
491 shallow_cmd.append(b)
492 shallow_cmd.extend(shallow_revisions)
493 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
494 656
495 def unpack(self, ud, destdir, d): 657 def unpack(self, ud, destdir, d):
496 """ unpack the downloaded src to destdir""" 658 """ unpack the downloaded src to destdir"""
497 659
498 subdir = ud.parm.get("subpath", "") 660 subdir = ud.parm.get("subdir")
499 if subdir != "": 661 subpath = ud.parm.get("subpath")
500 readpathspec = ":%s" % subdir 662 readpathspec = ""
501 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) 663 def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/"
502 else: 664
503 readpathspec = "" 665 if subpath:
504 def_destsuffix = "git/" 666 readpathspec = ":%s" % subpath
667 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
668
669 if subdir:
670 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
671 if os.path.isabs(subdir):
672 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
673 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
674 destdir = subdir
675 else:
676 destdir = os.path.join(destdir, subdir)
677 def_destsuffix = ""
505 678
506 destsuffix = ud.parm.get("destsuffix", def_destsuffix) 679 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
507 destdir = ud.destdir = os.path.join(destdir, destsuffix) 680 destdir = ud.destdir = os.path.join(destdir, destsuffix)
508 if os.path.exists(destdir): 681 if os.path.exists(destdir):
509 bb.utils.prunedir(destdir) 682 bb.utils.prunedir(destdir)
683 if not ud.bareclone:
684 ud.unpack_tracer.unpack("git", destdir)
510 685
511 need_lfs = self._need_lfs(ud) 686 need_lfs = self._need_lfs(ud)
512 687
@@ -516,13 +691,12 @@ class Git(FetchMethod):
516 source_found = False 691 source_found = False
517 source_error = [] 692 source_error = []
518 693
519 if not source_found: 694 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
520 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) 695 if clonedir_is_up_to_date:
521 if clonedir_is_up_to_date: 696 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
522 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) 697 source_found = True
523 source_found = True 698 else:
524 else: 699 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
525 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
526 700
527 if not source_found: 701 if not source_found:
528 if ud.shallow: 702 if ud.shallow:
@@ -538,28 +712,43 @@ class Git(FetchMethod):
538 if not source_found: 712 if not source_found:
539 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) 713 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
540 714
715 # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
716 # matches the revision
717 if 'tag' in ud.parm and sha1_re.match(ud.revision):
718 output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
719 output = output.strip()
720 if output != ud.revision:
721 # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list
722 # If it resolves to the same thing there isn't a problem.
723 output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
724 output2 = output2.strip()
725 if output != output2:
726 raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
727
541 repourl = self._get_repo_url(ud) 728 repourl = self._get_repo_url(ud)
542 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) 729 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
543 730
544 if self._contains_lfs(ud, d, destdir): 731 if self._contains_lfs(ud, d, destdir):
545 if need_lfs and not self._find_git_lfs(d): 732 if not need_lfs:
546 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
547 elif not need_lfs:
548 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) 733 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
734 else:
735 self._ensure_git_lfs(d, ud)
736
737 runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
549 738
550 if not ud.nocheckout: 739 if not ud.nocheckout:
551 if subdir != "": 740 if subpath:
552 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 741 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
553 workdir=destdir) 742 workdir=destdir)
554 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 743 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
555 elif not ud.nobranch: 744 elif not ud.nobranch:
556 branchname = ud.branches[ud.names[0]] 745 branchname = ud.branch
557 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ 746 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
558 ud.revisions[ud.names[0]]), d, workdir=destdir) 747 ud.revision), d, workdir=destdir)
559 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ 748 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
560 branchname), d, workdir=destdir) 749 branchname), d, workdir=destdir)
561 else: 750 else:
562 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) 751 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
563 752
564 return True 753 return True
565 754
@@ -573,8 +762,13 @@ class Git(FetchMethod):
573 clonedir = os.path.realpath(ud.localpath) 762 clonedir = os.path.realpath(ud.localpath)
574 to_remove.append(clonedir) 763 to_remove.append(clonedir)
575 764
765 # Remove shallow mirror tarball
766 if ud.shallow:
767 to_remove.append(ud.fullshallow)
768 to_remove.append(ud.fullshallow + ".done")
769
576 for r in to_remove: 770 for r in to_remove:
577 if os.path.exists(r): 771 if os.path.exists(r) or os.path.islink(r):
578 bb.note('Removing %s' % r) 772 bb.note('Removing %s' % r)
579 bb.utils.remove(r, True) 773 bb.utils.remove(r, True)
580 774
@@ -585,10 +779,10 @@ class Git(FetchMethod):
585 cmd = "" 779 cmd = ""
586 if ud.nobranch: 780 if ud.nobranch:
587 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( 781 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
588 ud.basecmd, ud.revisions[name]) 782 ud.basecmd, ud.revision)
589 else: 783 else:
590 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( 784 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
591 ud.basecmd, ud.revisions[name], ud.branches[name]) 785 ud.basecmd, ud.revision, ud.branch)
592 try: 786 try:
593 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 787 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
594 except bb.fetch2.FetchError: 788 except bb.fetch2.FetchError:
@@ -597,6 +791,37 @@ class Git(FetchMethod):
597 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 791 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
598 return output.split()[0] != "0" 792 return output.split()[0] != "0"
599 793
794 def _lfs_objects_downloaded(self, ud, d, wd):
795 """
796 Verifies whether the LFS objects for requested revisions have already been downloaded
797 """
798 # Bail out early if this repository doesn't use LFS
799 if not self._contains_lfs(ud, d, wd):
800 return True
801
802 self._ensure_git_lfs(d, ud)
803
804 # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
805 # existence.
806 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
807 cmd = "%s lfs ls-files -l %s" \
808 % (ud.basecmd, ud.revision)
809 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
810 # Do not do any further matching if no objects are managed by LFS
811 if not output:
812 return True
813
814 # Match all lines beginning with the hexadecimal OID
815 oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
816 for line in output.split("\n"):
817 oid = re.search(oid_regex, line)
818 if not oid:
819 bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
820 if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
821 return False
822
823 return True
824
600 def _need_lfs(self, ud): 825 def _need_lfs(self, ud):
601 return ud.parm.get("lfs", "1") == "1" 826 return ud.parm.get("lfs", "1") == "1"
602 827
@@ -604,20 +829,8 @@ class Git(FetchMethod):
604 """ 829 """
605 Check if the repository has 'lfs' (large file) content 830 Check if the repository has 'lfs' (large file) content
606 """ 831 """
607
608 if not ud.nobranch:
609 branchname = ud.branches[ud.names[0]]
610 else:
611 branchname = "master"
612
613 # The bare clonedir doesn't use the remote names; it has the branch immediately.
614 if wd == ud.clonedir:
615 refname = ud.branches[ud.names[0]]
616 else:
617 refname = "origin/%s" % ud.branches[ud.names[0]]
618
619 cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( 832 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
620 ud.basecmd, refname) 833 ud.basecmd, ud.revision)
621 834
622 try: 835 try:
623 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 836 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
@@ -627,12 +840,14 @@ class Git(FetchMethod):
627 pass 840 pass
628 return False 841 return False
629 842
630 def _find_git_lfs(self, d): 843 def _ensure_git_lfs(self, d, ud):
631 """ 844 """
632 Return True if git-lfs can be found, False otherwise. 845 Ensures that git-lfs is available, raising a FetchError if it isn't.
633 """ 846 """
634 import shutil 847 if shutil.which("git-lfs", path=d.getVar('PATH')) is None:
635 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None 848 raise bb.fetch2.FetchError(
849 "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 "
850 "to ignore it)" % self._get_repo_url(ud))
636 851
637 def _get_repo_url(self, ud): 852 def _get_repo_url(self, ud):
638 """ 853 """
@@ -640,22 +855,21 @@ class Git(FetchMethod):
640 """ 855 """
641 # Note that we do not support passwords directly in the git urls. There are several 856 # Note that we do not support passwords directly in the git urls. There are several
642 # reasons. SRC_URI can be written out to things like buildhistory and people don't 857 # reasons. SRC_URI can be written out to things like buildhistory and people don't
643 # want to leak passwords like that. Its also all too easy to share metadata without 858 # want to leak passwords like that. Its also all too easy to share metadata without
644 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as 859 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
645 # alternatives so we will not take patches adding password support here. 860 # alternatives so we will not take patches adding password support here.
646 if ud.user: 861 if ud.user:
647 username = ud.user + '@' 862 username = ud.user + '@'
648 else: 863 else:
649 username = "" 864 username = ""
650 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) 865 return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
651 866
652 def _revision_key(self, ud, d, name): 867 def _revision_key(self, ud, d, name):
653 """ 868 """
654 Return a unique key for the url 869 Return a unique key for the url
655 """ 870 """
656 # Collapse adjacent slashes 871 # Collapse adjacent slashes
657 slash_re = re.compile(r"/+") 872 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
658 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
659 873
660 def _lsremote(self, ud, d, search): 874 def _lsremote(self, ud, d, search):
661 """ 875 """
@@ -687,21 +901,27 @@ class Git(FetchMethod):
687 """ 901 """
688 Compute the HEAD revision for the url 902 Compute the HEAD revision for the url
689 """ 903 """
904 if not d.getVar("__BBSRCREV_SEEN"):
905 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
906
907 # Ensure we mark as not cached
908 bb.fetch2.mark_recipe_nocache(d)
909
690 output = self._lsremote(ud, d, "") 910 output = self._lsremote(ud, d, "")
691 # Tags of the form ^{} may not work, need to fallback to other form 911 # Tags of the form ^{} may not work, need to fallback to other form
692 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 912 if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
693 head = ud.unresolvedrev[name] 913 head = ud.unresolvedrev
694 tag = ud.unresolvedrev[name] 914 tag = ud.unresolvedrev
695 else: 915 else:
696 head = "refs/heads/%s" % ud.unresolvedrev[name] 916 head = "refs/heads/%s" % ud.unresolvedrev
697 tag = "refs/tags/%s" % ud.unresolvedrev[name] 917 tag = "refs/tags/%s" % ud.unresolvedrev
698 for s in [head, tag + "^{}", tag]: 918 for s in [head, tag + "^{}", tag]:
699 for l in output.strip().split('\n'): 919 for l in output.strip().split('\n'):
700 sha1, ref = l.split() 920 sha1, ref = l.split()
701 if s == ref: 921 if s == ref:
702 return sha1 922 return sha1
703 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ 923 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
704 (ud.unresolvedrev[name], ud.host+ud.path)) 924 (ud.unresolvedrev, ud.host+ud.path))
705 925
706 def latest_versionstring(self, ud, d): 926 def latest_versionstring(self, ud, d):
707 """ 927 """
@@ -711,60 +931,63 @@ class Git(FetchMethod):
711 """ 931 """
712 pupver = ('', '') 932 pupver = ('', '')
713 933
714 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
715 try: 934 try:
716 output = self._lsremote(ud, d, "refs/tags/*") 935 output = self._lsremote(ud, d, "refs/tags/*")
717 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: 936 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
718 bb.note("Could not list remote: %s" % str(e)) 937 bb.note("Could not list remote: %s" % str(e))
719 return pupver 938 return pupver
720 939
940 rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
941 pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
942 nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
943
721 verstring = "" 944 verstring = ""
722 revision = ""
723 for line in output.split("\n"): 945 for line in output.split("\n"):
724 if not line: 946 if not line:
725 break 947 break
726 948
727 tag_head = line.split("/")[-1] 949 m = rev_tag_re.match(line)
950 if not m:
951 continue
952
953 (revision, tag) = m.groups()
954
728 # Ignore non-released branches 955 # Ignore non-released branches
729 m = re.search(r"(alpha|beta|rc|final)+", tag_head) 956 if nonrel_re.search(tag):
730 if m:
731 continue 957 continue
732 958
733 # search for version in the line 959 # search for version in the line
734 tag = tagregex.search(tag_head) 960 m = pver_re.search(tag)
735 if tag is None: 961 if not m:
736 continue 962 continue
737 963
738 tag = tag.group('pver') 964 pver = m.group('pver').replace("_", ".")
739 tag = tag.replace("_", ".")
740 965
741 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: 966 if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
742 continue 967 continue
743 968
744 verstring = tag 969 verstring = pver
745 revision = line.split()[0]
746 pupver = (verstring, revision) 970 pupver = (verstring, revision)
747 971
748 return pupver 972 return pupver
749 973
750 def _build_revision(self, ud, d, name): 974 def _build_revision(self, ud, d, name):
751 return ud.revisions[name] 975 return ud.revision
752 976
753 def gitpkgv_revision(self, ud, d, name): 977 def gitpkgv_revision(self, ud, d, name):
754 """ 978 """
755 Return a sortable revision number by counting commits in the history 979 Return a sortable revision number by counting commits in the history
756 Based on gitpkgv.bblass in meta-openembedded 980 Based on gitpkgv.bblass in meta-openembedded
757 """ 981 """
758 rev = self._build_revision(ud, d, name) 982 rev = ud.revision
759 localpath = ud.localpath 983 localpath = ud.localpath
760 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) 984 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
761 if not os.path.exists(localpath): 985 if not os.path.exists(localpath):
762 commits = None 986 commits = None
763 else: 987 else:
764 if not os.path.exists(rev_file) or not os.path.getsize(rev_file): 988 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
765 from pipes import quote
766 commits = bb.fetch2.runfetchcmd( 989 commits = bb.fetch2.runfetchcmd(
767 "git rev-list %s -- | wc -l" % quote(rev), 990 "git rev-list %s -- | wc -l" % shlex.quote(rev),
768 d, quiet=True).strip().lstrip('0') 991 d, quiet=True).strip().lstrip('0')
769 if commits: 992 if commits:
770 open(rev_file, "w").write("%d\n" % int(commits)) 993 open(rev_file, "w").write("%d\n" % int(commits))
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index a4527bf364..5869e1b99b 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -62,36 +62,35 @@ class GitSM(Git):
62 return modules 62 return modules
63 63
64 # Collect the defined submodules, and their attributes 64 # Collect the defined submodules, and their attributes
65 for name in ud.names: 65 try:
66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
67 except:
68 # No submodules to update
69 gitmodules = ""
70
71 for m, md in parse_gitmodules(gitmodules).items():
66 try: 72 try:
67 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
68 except: 74 except:
69 # No submodules to update 75 # If the command fails, we don't have a valid file to check. If it doesn't
76 # fail -- it still might be a failure, see next check...
77 module_hash = ""
78
79 if not module_hash:
80 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
70 continue 81 continue
71 82
72 for m, md in parse_gitmodules(gitmodules).items(): 83 submodules.append(m)
73 try: 84 paths[m] = md['path']
74 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 85 revision[m] = ud.revision
75 except: 86 uris[m] = md['url']
76 # If the command fails, we don't have a valid file to check. If it doesn't 87 subrevision[m] = module_hash.split()[2]
77 # fail -- it still might be a failure, see next check... 88
78 module_hash = "" 89 # Convert relative to absolute uri based on parent uri
79 90 if uris[m].startswith('..') or uris[m].startswith('./'):
80 if not module_hash: 91 newud = copy.copy(ud)
81 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 92 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
82 continue 93 uris[m] = Git._get_repo_url(self, newud)
83
84 submodules.append(m)
85 paths[m] = md['path']
86 revision[m] = ud.revisions[name]
87 uris[m] = md['url']
88 subrevision[m] = module_hash.split()[2]
89
90 # Convert relative to absolute uri based on parent uri
91 if uris[m].startswith('..'):
92 newud = copy.copy(ud)
93 newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
94 uris[m] = Git._get_repo_url(self, newud)
95 94
96 for module in submodules: 95 for module in submodules:
97 # Translate the module url into a SRC_URI 96 # Translate the module url into a SRC_URI
@@ -115,10 +114,21 @@ class GitSM(Git):
115 # This has to be a file reference 114 # This has to be a file reference
116 proto = "file" 115 proto = "file"
117 url = "gitsm://" + uris[module] 116 url = "gitsm://" + uris[module]
117 if url.endswith("{}{}".format(ud.host, ud.path)):
118 raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
119 "Consider using git fetcher instead.")
118 120
119 url += ';protocol=%s' % proto 121 url += ';protocol=%s' % proto
120 url += ";name=%s" % module 122 url += ";name=%s" % module
121 url += ";subpath=%s" % module 123 url += ";subpath=%s" % module
124 url += ";nobranch=1"
125 url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0")
126 # Note that adding "user=" here to give credentials to the
127 # submodule is not supported. Since using SRC_URI to give git://
128 # URL a password is not supported, one have to use one of the
129 # recommended way (eg. ~/.netrc or SSH config) which does specify
130 # the user (See comment in git.py).
131 # So, we will not take patches adding "user=" support here.
122 132
123 ld = d.createCopy() 133 ld = d.createCopy()
124 # Not necessary to set SRC_URI, since we're passing the URI to 134 # Not necessary to set SRC_URI, since we're passing the URI to
@@ -136,20 +146,26 @@ class GitSM(Git):
136 146
137 return submodules != [] 147 return submodules != []
138 148
149 def call_process_submodules(self, ud, d, extra_check, subfunc):
150 # If we're using a shallow mirror tarball it needs to be
151 # unpacked temporarily so that we can examine the .gitmodules file
152 # Unpack even when ud.clonedir is not available,
153 # which may occur during a fast shallow clone
154 unpack = extra_check or not os.path.exists(ud.clonedir)
155 if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
156 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
157 try:
158 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
159 self.process_submodules(ud, tmpdir, subfunc, d)
160 finally:
161 shutil.rmtree(tmpdir)
162 else:
163 self.process_submodules(ud, ud.clonedir, subfunc, d)
164
139 def need_update(self, ud, d): 165 def need_update(self, ud, d):
140 if Git.need_update(self, ud, d): 166 if Git.need_update(self, ud, d):
141 return True 167 return True
142 168
143 try:
144 # Check for the nugget dropped by the download operation
145 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
146 (ud.basecmd), d, workdir=ud.clonedir)
147
148 if ud.revisions[ud.names[0]] in known_srcrevs.split():
149 return False
150 except bb.fetch2.FetchError:
151 pass
152
153 need_update_list = [] 169 need_update_list = []
154 def need_update_submodule(ud, url, module, modpath, workdir, d): 170 def need_update_submodule(ud, url, module, modpath, workdir, d):
155 url += ";bareclone=1;nobranch=1" 171 url += ";bareclone=1;nobranch=1"
@@ -163,22 +179,9 @@ class GitSM(Git):
163 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 179 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
164 need_update_result = True 180 need_update_result = True
165 181
166 # If we're using a shallow mirror tarball it needs to be unpacked 182 self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
167 # temporarily so that we can examine the .gitmodules file 183
168 if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): 184 if need_update_list:
169 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
170 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
171 self.process_submodules(ud, tmpdir, need_update_submodule, d)
172 shutil.rmtree(tmpdir)
173 else:
174 self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
175 if len(need_update_list) == 0:
176 # We already have the required commits of all submodules. Drop
177 # a nugget so we don't need to check again.
178 runfetchcmd("%s config --add bitbake.srcrev %s" % \
179 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
180
181 if len(need_update_list) > 0:
182 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 185 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
183 return True 186 return True
184 187
@@ -199,19 +202,7 @@ class GitSM(Git):
199 raise 202 raise
200 203
201 Git.download(self, ud, d) 204 Git.download(self, ud, d)
202 205 self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
203 # If we're using a shallow mirror tarball it needs to be unpacked
204 # temporarily so that we can examine the .gitmodules file
205 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
206 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
207 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
208 self.process_submodules(ud, tmpdir, download_submodule, d)
209 shutil.rmtree(tmpdir)
210 else:
211 self.process_submodules(ud, ud.clonedir, download_submodule, d)
212 # Drop a nugget for the srcrev we've fetched (used by need_update)
213 runfetchcmd("%s config --add bitbake.srcrev %s" % \
214 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
215 206
216 def unpack(self, ud, destdir, d): 207 def unpack(self, ud, destdir, d):
217 def unpack_submodules(ud, url, module, modpath, workdir, d): 208 def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -225,6 +216,10 @@ class GitSM(Git):
225 216
226 try: 217 try:
227 newfetch = Fetch([url], d, cache=False) 218 newfetch = Fetch([url], d, cache=False)
219 # modpath is needed by unpack tracer to calculate submodule
220 # checkout dir
221 new_ud = newfetch.ud[url]
222 new_ud.modpath = modpath
228 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 223 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
229 except Exception as e: 224 except Exception as e:
230 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 225 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
@@ -250,13 +245,27 @@ class GitSM(Git):
250 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 245 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
251 246
252 if not ud.bareclone and ret: 247 if not ud.bareclone and ret:
253 # All submodules should already be downloaded and configured in the tree. This simply sets 248 cmdprefix = ""
254 # up the configuration and checks out the files. The main project config should remain 249 # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed.
255 # unmodified, and no download from the internet should occur. 250 if not self._need_lfs(ud):
256 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 251 cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 "
252 runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir)
253 def clean(self, ud, d):
254 def clean_submodule(ud, url, module, modpath, workdir, d):
255 url += ";bareclone=1;nobranch=1"
256 try:
257 newfetch = Fetch([url], d, cache=False)
258 newfetch.clean()
259 except Exception as e:
260 logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
261
262 self.call_process_submodules(ud, d, True, clean_submodule)
263
264 # Clean top git dir
265 Git.clean(self, ud, d)
257 266
258 def implicit_urldata(self, ud, d): 267 def implicit_urldata(self, ud, d):
259 import shutil, subprocess, tempfile 268 import subprocess
260 269
261 urldata = [] 270 urldata = []
262 def add_submodule(ud, url, module, modpath, workdir, d): 271 def add_submodule(ud, url, module, modpath, workdir, d):
@@ -264,14 +273,6 @@ class GitSM(Git):
264 newfetch = Fetch([url], d, cache=False) 273 newfetch = Fetch([url], d, cache=False)
265 urldata.extend(newfetch.expanded_urldata()) 274 urldata.extend(newfetch.expanded_urldata())
266 275
267 # If we're using a shallow mirror tarball it needs to be unpacked 276 self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
268 # temporarily so that we can examine the .gitmodules file
269 if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
270 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
271 subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
272 self.process_submodules(ud, tmpdir, add_submodule, d)
273 shutil.rmtree(tmpdir)
274 else:
275 self.process_submodules(ud, ud.clonedir, add_submodule, d)
276 277
277 return urldata 278 return urldata
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py
new file mode 100644
index 0000000000..53c1d8d115
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gomod.py
@@ -0,0 +1,273 @@
1"""
2BitBake 'Fetch' implementation for Go modules
3
4The gomod/gomodgit fetchers are used to download Go modules to the module cache
5from a module proxy or directly from a version control repository.
6
7Example SRC_URI:
8
9SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
10SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
11
12Required SRC_URI parameters:
13
14- version
15 The version of the module.
16
17Optional SRC_URI parameters:
18
19- mod
20 Fetch and unpack the go.mod file only instead of the complete module.
21 The go command may need to download go.mod files for many different modules
22 when computing the build list, and go.mod files are much smaller than
23 module zip files.
24 The default is "0", set mod=1 for the go.mod file only.
25
26- sha256sum
27 The checksum of the module zip file, or the go.mod file in case of fetching
28 only the go.mod file. Alternatively, set the SRC_URI varible flag for
29 "module@version.sha256sum".
30
31- protocol
32 The method used when fetching directly from a version control repository.
33 The default is "https" for git.
34
35- repo
36 The URL when fetching directly from a version control repository. Required
37 when the URL is different from the module path.
38
39- srcrev
40 The revision identifier used when fetching directly from a version control
41 repository. Alternatively, set the SRCREV varible for "module@version".
42
43- subdir
44 The module subdirectory when fetching directly from a version control
45 repository. Required when the module is not located in the root of the
46 repository.
47
48Related variables:
49
50- GO_MOD_PROXY
51 The module proxy used by the fetcher.
52
53- GO_MOD_CACHE_DIR
54 The directory where the module cache is located.
55 This must match the exported GOMODCACHE variable for the go command to find
56 the downloaded modules.
57
58See the Go modules reference, https://go.dev/ref/mod, for more information
59about the module cache, module proxies and version control systems.
60"""
61
62import hashlib
63import os
64import re
65import shutil
66import subprocess
67import zipfile
68
69import bb
70from bb.fetch2 import FetchError
71from bb.fetch2 import MissingParameterError
72from bb.fetch2 import runfetchcmd
73from bb.fetch2 import subprocess_setup
74from bb.fetch2.git import Git
75from bb.fetch2.wget import Wget
76
77
78def escape(path):
79 """Escape capital letters using exclamation points."""
80 return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
81
82
83class GoMod(Wget):
84 """Class to fetch Go modules from a Go module proxy via wget"""
85
86 def supports(self, ud, d):
87 """Check to see if a given URL is for this fetcher."""
88 return ud.type == 'gomod'
89
90 def urldata_init(self, ud, d):
91 """Set up to download the module from the module proxy.
92
93 Set up to download the module zip file to the module cache directory
94 and unpack the go.mod file (unless downloading only the go.mod file):
95
96 cache/download/<module>/@v/<version>.zip: The module zip file.
97 cache/download/<module>/@v/<version>.mod: The go.mod file.
98 """
99
100 proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
101 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
102
103 if 'version' not in ud.parm:
104 raise MissingParameterError('version', ud.url)
105
106 module = ud.host
107 if ud.path != '/':
108 module += ud.path
109 ud.parm['module'] = module
110 version = ud.parm['version']
111
112 # Set URL and filename for wget download
113 if ud.parm.get('mod', '0') == '1':
114 ext = '.mod'
115 else:
116 ext = '.zip'
117 path = escape(f"{module}/@v/{version}{ext}")
118 ud.url = bb.fetch2.encodeurl(
119 ('https', proxy, '/' + path, None, None, None))
120 ud.parm['downloadfilename'] = f"{module.replace('/', '.')}@{version}{ext}"
121
122 # Set name for checksum verification
123 ud.parm['name'] = f"{module}@{version}"
124
125 # Set path for unpack
126 ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path)
127
128 super().urldata_init(ud, d)
129
130 def unpack(self, ud, rootdir, d):
131 """Unpack the module in the module cache."""
132
133 # Unpack the module zip file or go.mod file
134 unpackpath = os.path.join(rootdir, ud.parm['unpackpath'])
135 unpackdir = os.path.dirname(unpackpath)
136 bb.utils.mkdirhier(unpackdir)
137 ud.unpack_tracer.unpack("file-copy", unpackdir)
138 cmd = f"cp {ud.localpath} {unpackpath}"
139 path = d.getVar('PATH')
140 if path:
141 cmd = f"PATH={path} {cmd}"
142 name = os.path.basename(unpackpath)
143 bb.note(f"Unpacking {name} to {unpackdir}/")
144 subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup)
145
146 if name.endswith('.zip'):
147 # Unpack the go.mod file from the zip file
148 module = ud.parm['module']
149 name = name.rsplit('.', 1)[0] + '.mod'
150 bb.note(f"Unpacking {name} to {unpackdir}/")
151 with zipfile.ZipFile(ud.localpath) as zf:
152 with open(os.path.join(unpackdir, name), mode='wb') as mf:
153 try:
154 f = module + '@' + ud.parm['version'] + '/go.mod'
155 shutil.copyfileobj(zf.open(f), mf)
156 except KeyError:
157 # If the module does not have a go.mod file, synthesize
158 # one containing only a module statement.
159 mf.write(f'module {module}\n'.encode())
160
161
162class GoModGit(Git):
163 """Class to fetch Go modules directly from a git repository"""
164
165 def supports(self, ud, d):
166 """Check to see if a given URL is for this fetcher."""
167 return ud.type == 'gomodgit'
168
169 def urldata_init(self, ud, d):
170 """Set up to download the module from the git repository.
171
172 Set up to download the git repository to the module cache directory and
173 unpack the module zip file and the go.mod file:
174
175 cache/vcs/<hash>: The bare git repository.
176 cache/download/<module>/@v/<version>.zip: The module zip file.
177 cache/download/<module>/@v/<version>.mod: The go.mod file.
178 """
179
180 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
181
182 if 'version' not in ud.parm:
183 raise MissingParameterError('version', ud.url)
184
185 module = ud.host
186 if ud.path != '/':
187 module += ud.path
188 ud.parm['module'] = module
189
190 # Set host, path and srcrev for git download
191 if 'repo' in ud.parm:
192 repo = ud.parm['repo']
193 idx = repo.find('/')
194 if idx != -1:
195 ud.host = repo[:idx]
196 ud.path = repo[idx:]
197 else:
198 ud.host = repo
199 ud.path = ''
200 if 'protocol' not in ud.parm:
201 ud.parm['protocol'] = 'https'
202 ud.name = f"{module}@{ud.parm['version']}"
203 srcrev = d.getVar('SRCREV_' + ud.name)
204 if srcrev:
205 if 'srcrev' not in ud.parm:
206 ud.parm['srcrev'] = srcrev
207 else:
208 if 'srcrev' in ud.parm:
209 d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
210 if 'branch' not in ud.parm:
211 ud.parm['nobranch'] = '1'
212
213 # Set subpath, subdir and bareclone for git unpack
214 if 'subdir' in ud.parm:
215 ud.parm['subpath'] = ud.parm['subdir']
216 key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
217 ud.parm['key'] = key
218 ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
219 hashlib.sha256(key).hexdigest())
220 ud.parm['bareclone'] = '1'
221
222 super().urldata_init(ud, d)
223
224 def unpack(self, ud, rootdir, d):
225 """Unpack the module in the module cache."""
226
227 # Unpack the bare git repository
228 super().unpack(ud, rootdir, d)
229
230 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
231
232 # Create the info file
233 module = ud.parm['module']
234 repodir = os.path.join(rootdir, ud.parm['subdir'])
235 with open(repodir + '.info', 'wb') as f:
236 f.write(ud.parm['key'])
237
238 # Unpack the go.mod file from the repository
239 unpackdir = os.path.join(rootdir, moddir, 'cache/download',
240 escape(module), '@v')
241 bb.utils.mkdirhier(unpackdir)
242 srcrev = ud.parm['srcrev']
243 version = ud.parm['version']
244 escaped_version = escape(version)
245 cmd = f"git ls-tree -r --name-only '{srcrev}'"
246 if 'subpath' in ud.parm:
247 cmd += f" '{ud.parm['subpath']}'"
248 files = runfetchcmd(cmd, d, workdir=repodir).split()
249 name = escaped_version + '.mod'
250 bb.note(f"Unpacking {name} to {unpackdir}/")
251 with open(os.path.join(unpackdir, name), mode='wb') as mf:
252 f = 'go.mod'
253 if 'subpath' in ud.parm:
254 f = os.path.join(ud.parm['subpath'], f)
255 if f in files:
256 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
257 subprocess.check_call(cmd, stdout=mf, cwd=repodir,
258 preexec_fn=subprocess_setup)
259 else:
260 # If the module does not have a go.mod file, synthesize one
261 # containing only a module statement.
262 mf.write(f'module {module}\n'.encode())
263
264 # Synthesize the module zip file from the repository
265 name = escaped_version + '.zip'
266 bb.note(f"Unpacking {name} to {unpackdir}/")
267 with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
268 prefix = module + '@' + version + '/'
269 for f in files:
270 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
271 data = subprocess.check_output(cmd, cwd=repodir,
272 preexec_fn=subprocess_setup)
273 zf.writestr(prefix + f, data)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 063e13008a..cbff8c490c 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -242,6 +242,7 @@ class Hg(FetchMethod):
242 revflag = "-r %s" % ud.revision 242 revflag = "-r %s" % ud.revision
243 subdir = ud.parm.get("destsuffix", ud.module) 243 subdir = ud.parm.get("destsuffix", ud.module)
244 codir = "%s/%s" % (destdir, subdir) 244 codir = "%s/%s" % (destdir, subdir)
245 ud.unpack_tracer.unpack("hg", codir)
245 246
246 scmdata = ud.parm.get("scmdata", "") 247 scmdata = ud.parm.get("scmdata", "")
247 if scmdata != "nokeep": 248 if scmdata != "nokeep":
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index e7d1c8c58f..fda56a564e 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -29,11 +29,10 @@ class Local(FetchMethod):
29 29
30 def urldata_init(self, ud, d): 30 def urldata_init(self, ud, d):
31 # We don't set localfile as for this fetcher the file is already local! 31 # We don't set localfile as for this fetcher the file is already local!
32 ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0]) 32 ud.basename = os.path.basename(ud.path)
33 ud.basename = os.path.basename(ud.decodedurl) 33 ud.basepath = ud.path
34 ud.basepath = ud.decodedurl
35 ud.needdonestamp = False 34 ud.needdonestamp = False
36 if "*" in ud.decodedurl: 35 if "*" in ud.path:
37 raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url) 36 raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
38 return 37 return
39 38
@@ -41,28 +40,24 @@ class Local(FetchMethod):
41 """ 40 """
42 Return the local filename of a given url assuming a successful fetch. 41 Return the local filename of a given url assuming a successful fetch.
43 """ 42 """
44 return self.localpaths(urldata, d)[-1] 43 return self.localfile_searchpaths(urldata, d)[-1]
45 44
46 def localpaths(self, urldata, d): 45 def localfile_searchpaths(self, urldata, d):
47 """ 46 """
48 Return the local filename of a given url assuming a successful fetch. 47 Return the local filename of a given url assuming a successful fetch.
49 """ 48 """
50 searched = [] 49 searched = []
51 path = urldata.decodedurl 50 path = urldata.path
52 newpath = path 51 newpath = path
53 if path[0] == "/": 52 if path[0] == "/":
53 logger.debug2("Using absolute %s" % (path))
54 return [path] 54 return [path]
55 filespath = d.getVar('FILESPATH') 55 filespath = d.getVar('FILESPATH')
56 if filespath: 56 if filespath:
57 logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) 57 logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":"))))
58 newpath, hist = bb.utils.which(filespath, path, history=True) 58 newpath, hist = bb.utils.which(filespath, path, history=True)
59 logger.debug2("Using %s for %s" % (newpath, path))
59 searched.extend(hist) 60 searched.extend(hist)
60 if not os.path.exists(newpath):
61 dldirfile = os.path.join(d.getVar("DL_DIR"), path)
62 logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
63 bb.utils.mkdirhier(os.path.dirname(dldirfile))
64 searched.append(dldirfile)
65 return searched
66 return searched 61 return searched
67 62
68 def need_update(self, ud, d): 63 def need_update(self, ud, d):
@@ -78,9 +73,7 @@ class Local(FetchMethod):
78 filespath = d.getVar('FILESPATH') 73 filespath = d.getVar('FILESPATH')
79 if filespath: 74 if filespath:
80 locations = filespath.split(":") 75 locations = filespath.split(":")
81 locations.append(d.getVar("DL_DIR")) 76 msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n " + "\n ".join(locations)
82
83 msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations)
84 raise FetchError(msg) 77 raise FetchError(msg)
85 78
86 return True 79 return True
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 47898509ff..e469d66768 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -42,19 +42,27 @@ from bb.utils import is_semver
42 42
43def npm_package(package): 43def npm_package(package):
44 """Convert the npm package name to remove unsupported character""" 44 """Convert the npm package name to remove unsupported character"""
45 # Scoped package names (with the @) use the same naming convention 45 # For scoped package names ('@user/package') the '/' is replaced by a '-'.
46 # as the 'npm pack' command. 46 # This is similar to what 'npm pack' does, but 'npm pack' also strips the
47 if package.startswith("@"): 47 # leading '@', which can lead to ambiguous package names.
48 return re.sub("/", "-", package[1:]) 48 name = re.sub("/", "-", package)
49 return package 49 name = name.lower()
50 name = re.sub(r"[^\-a-z0-9@]", "", name)
51 name = name.strip("-")
52 return name
53
50 54
51def npm_filename(package, version): 55def npm_filename(package, version):
52 """Get the filename of a npm package""" 56 """Get the filename of a npm package"""
53 return npm_package(package) + "-" + version + ".tgz" 57 return npm_package(package) + "-" + version + ".tgz"
54 58
55def npm_localfile(package, version): 59def npm_localfile(package, version=None):
56 """Get the local filename of a npm package""" 60 """Get the local filename of a npm package"""
57 return os.path.join("npm2", npm_filename(package, version)) 61 if version is not None:
62 filename = npm_filename(package, version)
63 else:
64 filename = package
65 return os.path.join("npm2", filename)
58 66
59def npm_integrity(integrity): 67def npm_integrity(integrity):
60 """ 68 """
@@ -69,41 +77,67 @@ def npm_unpack(tarball, destdir, d):
69 bb.utils.mkdirhier(destdir) 77 bb.utils.mkdirhier(destdir)
70 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 78 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
71 cmd += " --no-same-owner" 79 cmd += " --no-same-owner"
80 cmd += " --delay-directory-restore"
72 cmd += " --strip-components=1" 81 cmd += " --strip-components=1"
73 runfetchcmd(cmd, d, workdir=destdir) 82 runfetchcmd(cmd, d, workdir=destdir)
83 runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
74 84
75class NpmEnvironment(object): 85class NpmEnvironment(object):
76 """ 86 """
77 Using a npm config file seems more reliable than using cli arguments. 87 Using a npm config file seems more reliable than using cli arguments.
78 This class allows to create a controlled environment for npm commands. 88 This class allows to create a controlled environment for npm commands.
79 """ 89 """
80 def __init__(self, d, configs=None): 90 def __init__(self, d, configs=[], npmrc=None):
81 self.d = d 91 self.d = d
82 self.configs = configs 92
93 self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
94
95 hn = self._home_npmrc(d)
96 if hn is not None:
97 with open(hn, 'r') as hnf:
98 self.user_config.write(hnf.read())
99
100 for key, value in configs:
101 self.user_config.write("%s=%s\n" % (key, value))
102
103 if npmrc:
104 self.global_config_name = npmrc
105 else:
106 self.global_config_name = "/dev/null"
107
108 def __del__(self):
109 if self.user_config:
110 self.user_config.close()
111
112 def _home_npmrc(self, d):
113 """Function to return user's HOME .npmrc file (or None if it doesn't exist)"""
114 home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc")
115 if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file):
116 bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\
117 f"npm fetcher will use {home_npmrc_file}")
118 return home_npmrc_file
119 return None
83 120
84 def run(self, cmd, args=None, configs=None, workdir=None): 121 def run(self, cmd, args=None, configs=None, workdir=None):
85 """Run npm command in a controlled environment""" 122 """Run npm command in a controlled environment"""
86 with tempfile.TemporaryDirectory() as tmpdir: 123 with tempfile.TemporaryDirectory() as tmpdir:
87 d = bb.data.createCopy(self.d) 124 d = bb.data.createCopy(self.d)
125 d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching
88 d.setVar("HOME", tmpdir) 126 d.setVar("HOME", tmpdir)
89 127
90 cfgfile = os.path.join(tmpdir, "npmrc")
91
92 if not workdir: 128 if not workdir:
93 workdir = tmpdir 129 workdir = tmpdir
94 130
95 def _run(cmd): 131 def _run(cmd):
96 cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd 132 cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
97 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd 133 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
98 return runfetchcmd(cmd, d, workdir=workdir) 134 return runfetchcmd(cmd, d, workdir=workdir)
99 135
100 if self.configs:
101 for key, value in self.configs:
102 _run("npm config set %s %s" % (key, shlex.quote(value)))
103
104 if configs: 136 if configs:
137 bb.warn("Use of configs argument of NpmEnvironment.run() function"
138 " is deprecated. Please use args argument instead.")
105 for key, value in configs: 139 for key, value in configs:
106 _run("npm config set %s %s" % (key, shlex.quote(value))) 140 cmd += " --%s=%s" % (key, shlex.quote(value))
107 141
108 if args: 142 if args:
109 for key, value in args: 143 for key, value in args:
@@ -142,12 +176,12 @@ class Npm(FetchMethod):
142 raise ParameterError("Invalid 'version' parameter", ud.url) 176 raise ParameterError("Invalid 'version' parameter", ud.url)
143 177
144 # Extract the 'registry' part of the url 178 # Extract the 'registry' part of the url
145 ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) 179 ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
146 180
147 # Using the 'downloadfilename' parameter as local filename 181 # Using the 'downloadfilename' parameter as local filename
148 # or the npm package name. 182 # or the npm package name.
149 if "downloadfilename" in ud.parm: 183 if "downloadfilename" in ud.parm:
150 ud.localfile = d.expand(ud.parm["downloadfilename"]) 184 ud.localfile = npm_localfile(ud.parm["downloadfilename"])
151 else: 185 else:
152 ud.localfile = npm_localfile(ud.package, ud.version) 186 ud.localfile = npm_localfile(ud.package, ud.version)
153 187
@@ -165,14 +199,14 @@ class Npm(FetchMethod):
165 199
166 def _resolve_proxy_url(self, ud, d): 200 def _resolve_proxy_url(self, ud, d):
167 def _npm_view(): 201 def _npm_view():
168 configs = [] 202 args = []
169 configs.append(("json", "true")) 203 args.append(("json", "true"))
170 configs.append(("registry", ud.registry)) 204 args.append(("registry", ud.registry))
171 pkgver = shlex.quote(ud.package + "@" + ud.version) 205 pkgver = shlex.quote(ud.package + "@" + ud.version)
172 cmd = ud.basecmd + " view %s" % pkgver 206 cmd = ud.basecmd + " view %s" % pkgver
173 env = NpmEnvironment(d) 207 env = NpmEnvironment(d)
174 check_network_access(d, cmd, ud.registry) 208 check_network_access(d, cmd, ud.registry)
175 view_string = env.run(cmd, configs=configs) 209 view_string = env.run(cmd, args=args)
176 210
177 if not view_string: 211 if not view_string:
178 raise FetchError("Unavailable package %s" % pkgver, ud.url) 212 raise FetchError("Unavailable package %s" % pkgver, ud.url)
@@ -280,6 +314,7 @@ class Npm(FetchMethod):
280 destsuffix = ud.parm.get("destsuffix", "npm") 314 destsuffix = ud.parm.get("destsuffix", "npm")
281 destdir = os.path.join(rootdir, destsuffix) 315 destdir = os.path.join(rootdir, destsuffix)
282 npm_unpack(ud.localpath, destdir, d) 316 npm_unpack(ud.localpath, destdir, d)
317 ud.unpack_tracer.unpack("npm", destdir)
283 318
284 def clean(self, ud, d): 319 def clean(self, ud, d):
285 """Clean any existing full or partial download""" 320 """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index 0c3511d8ab..2f9599ee9e 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -24,34 +24,39 @@ import bb
24from bb.fetch2 import Fetch 24from bb.fetch2 import Fetch
25from bb.fetch2 import FetchMethod 25from bb.fetch2 import FetchMethod
26from bb.fetch2 import ParameterError 26from bb.fetch2 import ParameterError
27from bb.fetch2 import runfetchcmd
27from bb.fetch2 import URI 28from bb.fetch2 import URI
28from bb.fetch2.npm import npm_integrity 29from bb.fetch2.npm import npm_integrity
29from bb.fetch2.npm import npm_localfile 30from bb.fetch2.npm import npm_localfile
30from bb.fetch2.npm import npm_unpack 31from bb.fetch2.npm import npm_unpack
31from bb.utils import is_semver 32from bb.utils import is_semver
33from bb.utils import lockfile
34from bb.utils import unlockfile
32 35
33def foreach_dependencies(shrinkwrap, callback=None, dev=False): 36def foreach_dependencies(shrinkwrap, callback=None, dev=False):
34 """ 37 """
35 Run a callback for each dependencies of a shrinkwrap file. 38 Run a callback for each dependencies of a shrinkwrap file.
36 The callback is using the format: 39 The callback is using the format:
37 callback(name, params, deptree) 40 callback(name, data, location)
38 with: 41 with:
39 name = the package name (string) 42 name = the package name (string)
40 params = the package parameters (dictionary) 43 data = the package data (dictionary)
41 deptree = the package dependency tree (array of strings) 44 location = the location of the package (string)
42 """ 45 """
43 def _walk_deps(deps, deptree): 46 packages = shrinkwrap.get("packages")
44 for name in deps: 47 if not packages:
45 subtree = [*deptree, name] 48 raise FetchError("Invalid shrinkwrap file format")
46 _walk_deps(deps[name].get("dependencies", {}), subtree) 49
47 if callback is not None: 50 for location, data in packages.items():
48 if deps[name].get("dev", False) and not dev: 51 # Skip empty main and local link target packages
49 continue 52 if not location.startswith('node_modules/'):
50 elif deps[name].get("bundled", False): 53 continue
51 continue 54 elif not dev and data.get("dev", False):
52 callback(name, deps[name], subtree) 55 continue
53 56 elif data.get("inBundle", False):
54 _walk_deps(shrinkwrap.get("dependencies", {}), []) 57 continue
58 name = location.split('node_modules/')[-1]
59 callback(name, data, location)
55 60
56class NpmShrinkWrap(FetchMethod): 61class NpmShrinkWrap(FetchMethod):
57 """Class to fetch all package from a shrinkwrap file""" 62 """Class to fetch all package from a shrinkwrap file"""
@@ -72,19 +77,28 @@ class NpmShrinkWrap(FetchMethod):
72 # Resolve the dependencies 77 # Resolve the dependencies
73 ud.deps = [] 78 ud.deps = []
74 79
75 def _resolve_dependency(name, params, deptree): 80 def _resolve_dependency(name, params, destsuffix):
76 url = None 81 url = None
77 localpath = None 82 localpath = None
78 extrapaths = [] 83 extrapaths = []
79 destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] 84 unpack = True
80 destsuffix = os.path.join(*destsubdirs)
81 85
82 integrity = params.get("integrity", None) 86 integrity = params.get("integrity")
83 resolved = params.get("resolved", None) 87 resolved = params.get("resolved")
84 version = params.get("version", None) 88 version = params.get("version")
89 link = params.get("link", False)
90
91 # Handle link sources
92 if link:
93 localpath = resolved
94 unpack = False
85 95
86 # Handle registry sources 96 # Handle registry sources
87 if is_semver(version) and resolved and integrity: 97 elif version and is_semver(version) and integrity:
98 # Handle duplicate dependencies without url
99 if not resolved:
100 return
101
88 localfile = npm_localfile(name, version) 102 localfile = npm_localfile(name, version)
89 103
90 uri = URI(resolved) 104 uri = URI(resolved)
@@ -108,10 +122,10 @@ class NpmShrinkWrap(FetchMethod):
108 extrapaths.append(resolvefile) 122 extrapaths.append(resolvefile)
109 123
110 # Handle http tarball sources 124 # Handle http tarball sources
111 elif version.startswith("http") and integrity: 125 elif resolved.startswith("http") and integrity:
112 localfile = os.path.join("npm2", os.path.basename(version)) 126 localfile = npm_localfile(os.path.basename(resolved))
113 127
114 uri = URI(version) 128 uri = URI(resolved)
115 uri.params["downloadfilename"] = localfile 129 uri.params["downloadfilename"] = localfile
116 130
117 checksum_name, checksum_expected = npm_integrity(integrity) 131 checksum_name, checksum_expected = npm_integrity(integrity)
@@ -121,8 +135,12 @@ class NpmShrinkWrap(FetchMethod):
121 135
122 localpath = os.path.join(d.getVar("DL_DIR"), localfile) 136 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
123 137
138 # Handle local tarball sources
139 elif resolved.startswith("file"):
140 localpath = resolved[5:]
141
124 # Handle git sources 142 # Handle git sources
125 elif version.startswith("git"): 143 elif resolved.startswith("git"):
126 regex = re.compile(r""" 144 regex = re.compile(r"""
127 ^ 145 ^
128 git\+ 146 git\+
@@ -134,29 +152,31 @@ class NpmShrinkWrap(FetchMethod):
134 $ 152 $
135 """, re.VERBOSE) 153 """, re.VERBOSE)
136 154
137 match = regex.match(version) 155 match = regex.match(resolved)
138
139 if not match: 156 if not match:
140 raise ParameterError("Invalid git url: %s" % version, ud.url) 157 raise ParameterError("Invalid git url: %s" % resolved, ud.url)
141 158
142 groups = match.groupdict() 159 groups = match.groupdict()
143 160
144 uri = URI("git://" + str(groups["url"])) 161 uri = URI("git://" + str(groups["url"]))
145 uri.params["protocol"] = str(groups["protocol"]) 162 uri.params["protocol"] = str(groups["protocol"])
146 uri.params["rev"] = str(groups["rev"]) 163 uri.params["rev"] = str(groups["rev"])
164 uri.params["nobranch"] = "1"
147 uri.params["destsuffix"] = destsuffix 165 uri.params["destsuffix"] = destsuffix
148 166
149 url = str(uri) 167 url = str(uri)
150 168
151 # local tarball sources and local link sources are unsupported
152 else: 169 else:
153 raise ParameterError("Unsupported dependency: %s" % name, ud.url) 170 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
154 171
172 # name is needed by unpack tracer for module mapping
155 ud.deps.append({ 173 ud.deps.append({
174 "name": name,
156 "url": url, 175 "url": url,
157 "localpath": localpath, 176 "localpath": localpath,
158 "extrapaths": extrapaths, 177 "extrapaths": extrapaths,
159 "destsuffix": destsuffix, 178 "destsuffix": destsuffix,
179 "unpack": unpack,
160 }) 180 })
161 181
162 try: 182 try:
@@ -177,17 +197,23 @@ class NpmShrinkWrap(FetchMethod):
177 # This fetcher resolves multiple URIs from a shrinkwrap file and then 197 # This fetcher resolves multiple URIs from a shrinkwrap file and then
178 # forwards it to a proxy fetcher. The management of the donestamp file, 198 # forwards it to a proxy fetcher. The management of the donestamp file,
179 # the lockfile and the checksums are forwarded to the proxy fetcher. 199 # the lockfile and the checksums are forwarded to the proxy fetcher.
180 ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) 200 shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
201 if shrinkwrap_urls:
202 ud.proxy = Fetch(shrinkwrap_urls, data)
181 ud.needdonestamp = False 203 ud.needdonestamp = False
182 204
183 @staticmethod 205 @staticmethod
184 def _foreach_proxy_method(ud, handle): 206 def _foreach_proxy_method(ud, handle):
185 returns = [] 207 returns = []
186 for proxy_url in ud.proxy.urls: 208 #Check if there are dependencies before try to fetch them
187 proxy_ud = ud.proxy.ud[proxy_url] 209 if len(ud.deps) > 0:
188 proxy_d = ud.proxy.d 210 for proxy_url in ud.proxy.urls:
189 proxy_ud.setup_localpath(proxy_d) 211 proxy_ud = ud.proxy.ud[proxy_url]
190 returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) 212 proxy_d = ud.proxy.d
213 proxy_ud.setup_localpath(proxy_d)
214 lf = lockfile(proxy_ud.lockfile)
215 returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
216 unlockfile(lf)
191 return returns 217 return returns
192 218
193 def verify_donestamp(self, ud, d): 219 def verify_donestamp(self, ud, d):
@@ -220,10 +246,11 @@ class NpmShrinkWrap(FetchMethod):
220 246
221 def unpack(self, ud, rootdir, d): 247 def unpack(self, ud, rootdir, d):
222 """Unpack the downloaded dependencies""" 248 """Unpack the downloaded dependencies"""
223 destdir = d.getVar("S") 249 destdir = rootdir
224 destsuffix = ud.parm.get("destsuffix") 250 destsuffix = ud.parm.get("destsuffix")
225 if destsuffix: 251 if destsuffix:
226 destdir = os.path.join(rootdir, destsuffix) 252 destdir = os.path.join(rootdir, destsuffix)
253 ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
227 254
228 bb.utils.mkdirhier(destdir) 255 bb.utils.mkdirhier(destdir)
229 bb.utils.copyfile(ud.shrinkwrap_file, 256 bb.utils.copyfile(ud.shrinkwrap_file,
@@ -237,7 +264,16 @@ class NpmShrinkWrap(FetchMethod):
237 264
238 for dep in manual: 265 for dep in manual:
239 depdestdir = os.path.join(destdir, dep["destsuffix"]) 266 depdestdir = os.path.join(destdir, dep["destsuffix"])
240 npm_unpack(dep["localpath"], depdestdir, d) 267 if dep["url"]:
268 npm_unpack(dep["localpath"], depdestdir, d)
269 else:
270 depsrcdir= os.path.join(destdir, dep["localpath"])
271 if dep["unpack"]:
272 npm_unpack(depsrcdir, depdestdir, d)
273 else:
274 bb.utils.mkdirhier(depdestdir)
275 cmd = 'cp -fpPRH "%s/." .' % (depsrcdir)
276 runfetchcmd(cmd, d, workdir=depdestdir)
241 277
242 def clean(self, ud, d): 278 def clean(self, ud, d):
243 """Clean any existing full or partial download""" 279 """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index d9ce44390c..495ac8a30a 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright BitBake Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4""" 6"""
@@ -9,6 +11,7 @@ Based on the svn "Fetch" implementation.
9 11
10import logging 12import logging
11import os 13import os
14import re
12import bb 15import bb
13from bb.fetch2 import FetchMethod 16from bb.fetch2 import FetchMethod
14from bb.fetch2 import FetchError 17from bb.fetch2 import FetchError
@@ -36,6 +39,7 @@ class Osc(FetchMethod):
36 # Create paths to osc checkouts 39 # Create paths to osc checkouts
37 oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc") 40 oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
38 relpath = self._strip_leading_slashes(ud.path) 41 relpath = self._strip_leading_slashes(ud.path)
42 ud.oscdir = oscdir
39 ud.pkgdir = os.path.join(oscdir, ud.host) 43 ud.pkgdir = os.path.join(oscdir, ud.host)
40 ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) 44 ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
41 45
@@ -43,13 +47,13 @@ class Osc(FetchMethod):
43 ud.revision = ud.parm['rev'] 47 ud.revision = ud.parm['rev']
44 else: 48 else:
45 pv = d.getVar("PV", False) 49 pv = d.getVar("PV", False)
46 rev = bb.fetch2.srcrev_internal_helper(ud, d) 50 rev = bb.fetch2.srcrev_internal_helper(ud, d, '')
47 if rev: 51 if rev:
48 ud.revision = rev 52 ud.revision = rev
49 else: 53 else:
50 ud.revision = "" 54 ud.revision = ""
51 55
52 ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision)) 56 ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision))
53 57
54 def _buildosccommand(self, ud, d, command): 58 def _buildosccommand(self, ud, d, command):
55 """ 59 """
@@ -59,26 +63,49 @@ class Osc(FetchMethod):
59 63
60 basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc" 64 basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
61 65
62 proto = ud.parm.get('protocol', 'ocs') 66 proto = ud.parm.get('protocol', 'https')
63 67
64 options = [] 68 options = []
65 69
66 config = "-c %s" % self.generate_config(ud, d) 70 config = "-c %s" % self.generate_config(ud, d)
67 71
68 if ud.revision: 72 if getattr(ud, 'revision', ''):
69 options.append("-r %s" % ud.revision) 73 options.append("-r %s" % ud.revision)
70 74
71 coroot = self._strip_leading_slashes(ud.path) 75 coroot = self._strip_leading_slashes(ud.path)
72 76
73 if command == "fetch": 77 if command == "fetch":
74 osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) 78 osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options))
75 elif command == "update": 79 elif command == "update":
76 osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) 80 osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options))
81 elif command == "api_source":
82 osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module)
77 else: 83 else:
78 raise FetchError("Invalid osc command %s" % command, ud.url) 84 raise FetchError("Invalid osc command %s" % command, ud.url)
79 85
80 return osccmd 86 return osccmd
81 87
88 def _latest_revision(self, ud, d, name):
89 """
90 Fetch latest revision for the given package
91 """
92 api_source_cmd = self._buildosccommand(ud, d, "api_source")
93
94 output = runfetchcmd(api_source_cmd, d)
95 match = re.match(r'<directory ?.* rev="(\d+)".*>', output)
96 if match is None:
97 raise FetchError("Unable to parse osc response", ud.url)
98 return match.groups()[0]
99
100 def _revision_key(self, ud, d, name):
101 """
102 Return a unique key for the url
103 """
104 # Collapse adjacent slashes
105 slash_re = re.compile(r"/+")
106 rev = getattr(ud, 'revision', "latest")
107 return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev)
108
82 def download(self, ud, d): 109 def download(self, ud, d):
83 """ 110 """
84 Fetch url 111 Fetch url
@@ -86,7 +113,7 @@ class Osc(FetchMethod):
86 113
87 logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") 114 logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
88 115
89 if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK): 116 if os.access(ud.moddir, os.R_OK):
90 oscupdatecmd = self._buildosccommand(ud, d, "update") 117 oscupdatecmd = self._buildosccommand(ud, d, "update")
91 logger.info("Update "+ ud.url) 118 logger.info("Update "+ ud.url)
92 # update sources there 119 # update sources there
@@ -114,20 +141,23 @@ class Osc(FetchMethod):
114 Generate a .oscrc to be used for this run. 141 Generate a .oscrc to be used for this run.
115 """ 142 """
116 143
117 config_path = os.path.join(d.getVar('OSCDIR'), "oscrc") 144 config_path = os.path.join(ud.oscdir, "oscrc")
145 if not os.path.exists(ud.oscdir):
146 bb.utils.mkdirhier(ud.oscdir)
147
118 if (os.path.exists(config_path)): 148 if (os.path.exists(config_path)):
119 os.remove(config_path) 149 os.remove(config_path)
120 150
121 f = open(config_path, 'w') 151 f = open(config_path, 'w')
152 proto = ud.parm.get('protocol', 'https')
122 f.write("[general]\n") 153 f.write("[general]\n")
123 f.write("apisrv = %s\n" % ud.host) 154 f.write("apiurl = %s://%s\n" % (proto, ud.host))
124 f.write("scheme = http\n")
125 f.write("su-wrapper = su -c\n") 155 f.write("su-wrapper = su -c\n")
126 f.write("build-root = %s\n" % d.getVar('WORKDIR')) 156 f.write("build-root = %s\n" % d.getVar('WORKDIR'))
127 f.write("urllist = %s\n" % d.getVar("OSCURLLIST")) 157 f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
128 f.write("extra-pkgs = gzip\n") 158 f.write("extra-pkgs = gzip\n")
129 f.write("\n") 159 f.write("\n")
130 f.write("[%s]\n" % ud.host) 160 f.write("[%s://%s]\n" % (proto, ud.host))
131 f.write("user = %s\n" % ud.parm["user"]) 161 f.write("user = %s\n" % ud.parm["user"])
132 f.write("pass = %s\n" % ud.parm["pswd"]) 162 f.write("pass = %s\n" % ud.parm["pswd"])
133 f.close() 163 f.close()
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index e2a41a4a12..3b6fa4b1ec 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -134,7 +134,7 @@ class Perforce(FetchMethod):
134 134
135 ud.setup_revisions(d) 135 ud.setup_revisions(d)
136 136
137 ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision)) 137 ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
138 138
139 def _buildp4command(self, ud, d, command, depot_filename=None): 139 def _buildp4command(self, ud, d, command, depot_filename=None):
140 """ 140 """
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py
index ffca73c8e4..22c0538139 100644
--- a/bitbake/lib/bb/fetch2/s3.py
+++ b/bitbake/lib/bb/fetch2/s3.py
@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use.
18import os 18import os
19import bb 19import bb
20import urllib.request, urllib.parse, urllib.error 20import urllib.request, urllib.parse, urllib.error
21import re
21from bb.fetch2 import FetchMethod 22from bb.fetch2 import FetchMethod
22from bb.fetch2 import FetchError 23from bb.fetch2 import FetchError
23from bb.fetch2 import runfetchcmd 24from bb.fetch2 import runfetchcmd
24 25
26def convertToBytes(value, unit):
27 value = float(value)
28 if (unit == "KiB"):
29 value = value*1024.0;
30 elif (unit == "MiB"):
31 value = value*1024.0*1024.0;
32 elif (unit == "GiB"):
33 value = value*1024.0*1024.0*1024.0;
34 return value
35
36class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
37 """
38 Extract progress information from s3 cp output, e.g.:
39 Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
40 """
41 def __init__(self, d):
42 super(S3ProgressHandler, self).__init__(d)
43 # Send an initial progress event so the bar gets shown
44 self._fire_progress(0)
45
46 def writeline(self, line):
47 percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
48 if percs:
49 completed = (percs[-1][0])
50 completedUnit = (percs[-1][1])
51 total = (percs[-1][2])
52 totalUnit = (percs[-1][3])
53 completed = convertToBytes(completed, completedUnit)
54 total = convertToBytes(total, totalUnit)
55 progress = (completed/total)*100.0
56 rate = percs[-1][4]
57 self.update(progress, rate)
58 return False
59 return True
60
61
25class S3(FetchMethod): 62class S3(FetchMethod):
26 """Class to fetch urls via 'aws s3'""" 63 """Class to fetch urls via 'aws s3'"""
27 64
@@ -40,7 +77,7 @@ class S3(FetchMethod):
40 else: 77 else:
41 ud.basename = os.path.basename(ud.path) 78 ud.basename = os.path.basename(ud.path)
42 79
43 ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) 80 ud.localfile = ud.basename
44 81
45 ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3" 82 ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
46 83
@@ -52,7 +89,9 @@ class S3(FetchMethod):
52 89
53 cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) 90 cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
54 bb.fetch2.check_network_access(d, cmd, ud.url) 91 bb.fetch2.check_network_access(d, cmd, ud.url)
55 runfetchcmd(cmd, d) 92
93 progresshandler = S3ProgressHandler(d)
94 runfetchcmd(cmd, d, False, log=progresshandler)
56 95
57 # Additional sanity checks copied from the wget class (although there 96 # Additional sanity checks copied from the wget class (although there
58 # are no known issues which mean these are required, treat the aws cli 97 # are no known issues which mean these are required, treat the aws cli
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
index f87f292e5d..bee71a0d0d 100644
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ b/bitbake/lib/bb/fetch2/sftp.py
@@ -77,7 +77,7 @@ class SFTP(FetchMethod):
77 else: 77 else:
78 ud.basename = os.path.basename(ud.path) 78 ud.basename = os.path.basename(ud.path)
79 79
80 ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) 80 ud.localfile = ud.basename
81 81
82 def download(self, ud, d): 82 def download(self, ud, d):
83 """Fetch urls""" 83 """Fetch urls"""
@@ -103,7 +103,7 @@ class SFTP(FetchMethod):
103 if path[:3] == '/~/': 103 if path[:3] == '/~/':
104 path = path[3:] 104 path = path[3:]
105 105
106 remote = '%s%s:%s' % (user, urlo.hostname, path) 106 remote = '"%s%s:%s"' % (user, urlo.hostname, path)
107 107
108 cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) 108 cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
109 109
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 2c8557e1f8..2a0f2cb44b 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -32,6 +32,7 @@ IETF secsh internet draft:
32 32
33import re, os 33import re, os
34from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd 34from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
35import urllib
35 36
36 37
37__pattern__ = re.compile(r''' 38__pattern__ = re.compile(r'''
@@ -40,9 +41,9 @@ __pattern__ = re.compile(r'''
40 ( # Optional username/password block 41 ( # Optional username/password block
41 (?P<user>\S+) # username 42 (?P<user>\S+) # username
42 (:(?P<pass>\S+))? # colon followed by the password (optional) 43 (:(?P<pass>\S+))? # colon followed by the password (optional)
43 )?
44 (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) 44 (?P<cparam>(;[^;]+)*)? # connection parameters block (optional)
45 @ 45 @
46 )?
46 (?P<host>\S+?) # non-greedy match of the host 47 (?P<host>\S+?) # non-greedy match of the host
47 (:(?P<port>[0-9]+))? # colon followed by the port (optional) 48 (:(?P<port>[0-9]+))? # colon followed by the port (optional)
48 / 49 /
@@ -70,9 +71,9 @@ class SSH(FetchMethod):
70 "git:// prefix with protocol=ssh", urldata.url) 71 "git:// prefix with protocol=ssh", urldata.url)
71 m = __pattern__.match(urldata.url) 72 m = __pattern__.match(urldata.url)
72 path = m.group('path') 73 path = m.group('path')
74 path = urllib.parse.unquote(path)
73 host = m.group('host') 75 host = m.group('host')
74 urldata.localpath = os.path.join(d.getVar('DL_DIR'), 76 urldata.localfile = os.path.basename(os.path.normpath(path))
75 os.path.basename(os.path.normpath(path)))
76 77
77 def download(self, urldata, d): 78 def download(self, urldata, d):
78 dldir = d.getVar('DL_DIR') 79 dldir = d.getVar('DL_DIR')
@@ -96,6 +97,11 @@ class SSH(FetchMethod):
96 fr += '@%s' % host 97 fr += '@%s' % host
97 else: 98 else:
98 fr = host 99 fr = host
100
101 if path[0] != '~':
102 path = '/%s' % path
103 path = urllib.parse.unquote(path)
104
99 fr += ':%s' % path 105 fr += ':%s' % path
100 106
101 cmd = 'scp -B -r %s %s %s/' % ( 107 cmd = 'scp -B -r %s %s %s/' % (
@@ -108,3 +114,41 @@ class SSH(FetchMethod):
108 114
109 runfetchcmd(cmd, d) 115 runfetchcmd(cmd, d)
110 116
117 def checkstatus(self, fetch, urldata, d):
118 """
119 Check the status of the url
120 """
121 m = __pattern__.match(urldata.url)
122 path = m.group('path')
123 host = m.group('host')
124 port = m.group('port')
125 user = m.group('user')
126 password = m.group('pass')
127
128 if port:
129 portarg = '-P %s' % port
130 else:
131 portarg = ''
132
133 if user:
134 fr = user
135 if password:
136 fr += ':%s' % password
137 fr += '@%s' % host
138 else:
139 fr = host
140
141 if path[0] != '~':
142 path = '/%s' % path
143 path = urllib.parse.unquote(path)
144
145 cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % (
146 portarg,
147 fr,
148 path
149 )
150
151 check_network_access(d, cmd, urldata.url)
152 runfetchcmd(cmd, d)
153
154 return True
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 8856ef1c62..0852108e7d 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -57,7 +57,12 @@ class Svn(FetchMethod):
57 if 'rev' in ud.parm: 57 if 'rev' in ud.parm:
58 ud.revision = ud.parm['rev'] 58 ud.revision = ud.parm['rev']
59 59
60 ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision)) 60 # Whether to use the @REV peg-revision syntax in the svn command or not
61 ud.pegrevision = True
62 if 'nopegrevision' in ud.parm:
63 ud.pegrevision = False
64
65 ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision]))
61 66
62 def _buildsvncommand(self, ud, d, command): 67 def _buildsvncommand(self, ud, d, command):
63 """ 68 """
@@ -86,7 +91,7 @@ class Svn(FetchMethod):
86 if command == "info": 91 if command == "info":
87 svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) 92 svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
88 elif command == "log1": 93 elif command == "log1":
89 svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) 94 svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
90 else: 95 else:
91 suffix = "" 96 suffix = ""
92 97
@@ -98,7 +103,8 @@ class Svn(FetchMethod):
98 103
99 if ud.revision: 104 if ud.revision:
100 options.append("-r %s" % ud.revision) 105 options.append("-r %s" % ud.revision)
101 suffix = "@%s" % (ud.revision) 106 if ud.pegrevision:
107 suffix = "@%s" % (ud.revision)
102 108
103 if command == "fetch": 109 if command == "fetch":
104 transportuser = ud.parm.get("transportuser", "") 110 transportuser = ud.parm.get("transportuser", "")
@@ -204,3 +210,6 @@ class Svn(FetchMethod):
204 210
205 def _build_revision(self, ud, d): 211 def _build_revision(self, ud, d):
206 return ud.revision 212 return ud.revision
213
214 def supports_checksum(self, urldata):
215 return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 6d82f3af07..7e43d3bc97 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -26,7 +26,6 @@ from bb.fetch2 import FetchMethod
26from bb.fetch2 import FetchError 26from bb.fetch2 import FetchError
27from bb.fetch2 import logger 27from bb.fetch2 import logger
28from bb.fetch2 import runfetchcmd 28from bb.fetch2 import runfetchcmd
29from bb.utils import export_proxies
30from bs4 import BeautifulSoup 29from bs4 import BeautifulSoup
31from bs4 import SoupStrainer 30from bs4 import SoupStrainer
32 31
@@ -52,18 +51,19 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
52 51
53 52
54class Wget(FetchMethod): 53class Wget(FetchMethod):
54 """Class to fetch urls via 'wget'"""
55 55
56 # CDNs like CloudFlare may do a 'browser integrity test' which can fail 56 def check_certs(self, d):
57 # with the standard wget/urllib User-Agent, so pretend to be a modern 57 """
58 # browser. 58 Should certificates be checked?
59 user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" 59 """
60 return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
60 61
61 """Class to fetch urls via 'wget'"""
62 def supports(self, ud, d): 62 def supports(self, ud, d):
63 """ 63 """
64 Check to see if a given url can be fetched with wget. 64 Check to see if a given url can be fetched with wget.
65 """ 65 """
66 return ud.type in ['http', 'https', 'ftp'] 66 return ud.type in ['http', 'https', 'ftp', 'ftps']
67 67
68 def recommends_checksum(self, urldata): 68 def recommends_checksum(self, urldata):
69 return True 69 return True
@@ -78,11 +78,17 @@ class Wget(FetchMethod):
78 else: 78 else:
79 ud.basename = os.path.basename(ud.path) 79 ud.basename = os.path.basename(ud.path)
80 80
81 ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) 81 ud.localfile = ud.basename
82 if not ud.localfile: 82 if not ud.localfile:
83 ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) 83 ud.localfile = ud.host + ud.path.replace("/", ".")
84 84
85 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" 85 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100"
86
87 if ud.type == 'ftp' or ud.type == 'ftps':
88 self.basecmd += " --passive-ftp"
89
90 if not self.check_certs(d):
91 self.basecmd += " --no-check-certificate"
86 92
87 def _runwget(self, ud, d, command, quiet, workdir=None): 93 def _runwget(self, ud, d, command, quiet, workdir=None):
88 94
@@ -90,39 +96,53 @@ class Wget(FetchMethod):
90 96
91 logger.debug2("Fetching %s using command '%s'" % (ud.url, command)) 97 logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
92 bb.fetch2.check_network_access(d, command, ud.url) 98 bb.fetch2.check_network_access(d, command, ud.url)
93 runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir) 99 runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir)
94 100
95 def download(self, ud, d): 101 def download(self, ud, d):
96 """Fetch urls""" 102 """Fetch urls"""
97 103
98 fetchcmd = self.basecmd 104 fetchcmd = self.basecmd
99 105
100 if 'downloadfilename' in ud.parm: 106 dldir = os.path.realpath(d.getVar("DL_DIR"))
101 localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) 107 localpath = os.path.join(dldir, ud.localfile) + ".tmp"
102 bb.utils.mkdirhier(os.path.dirname(localpath)) 108 bb.utils.mkdirhier(os.path.dirname(localpath))
103 fetchcmd += " -O %s" % shlex.quote(localpath) 109 fetchcmd += " --output-document=%s" % shlex.quote(localpath)
104 110
105 if ud.user and ud.pswd: 111 if ud.user and ud.pswd:
106 fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) 112 fetchcmd += " --auth-no-challenge"
113 if ud.parm.get("redirectauth", "1") == "1":
114 # An undocumented feature of wget is that if the
115 # username/password are specified on the URI, wget will only
116 # send the Authorization header to the first host and not to
117 # any hosts that it is redirected to. With the increasing
118 # usage of temporary AWS URLs, this difference now matters as
119 # AWS will reject any request that has authentication both in
120 # the query parameters (from the redirect) and in the
121 # Authorization header.
122 fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
107 123
108 uri = ud.url.split(";")[0] 124 uri = ud.url.split(";")[0]
109 if os.path.exists(ud.localpath): 125 fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
110 # file exists, but we didnt complete it.. trying again..
111 fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
112 else:
113 fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
114
115 self._runwget(ud, d, fetchcmd, False) 126 self._runwget(ud, d, fetchcmd, False)
116 127
117 # Sanity check since wget can pretend it succeed when it didn't 128 # Sanity check since wget can pretend it succeed when it didn't
118 # Also, this used to happen if sourceforge sent us to the mirror page 129 # Also, this used to happen if sourceforge sent us to the mirror page
119 if not os.path.exists(ud.localpath): 130 if not os.path.exists(localpath):
120 raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) 131 raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
121 132
122 if os.path.getsize(ud.localpath) == 0: 133 if os.path.getsize(localpath) == 0:
123 os.remove(ud.localpath) 134 os.remove(localpath)
124 raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) 135 raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
125 136
137 # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
138 # original file, which might be a race (imagine two recipes referencing the same
139 # source, one with an incorrect checksum)
140 bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
141
142 # Remove the ".tmp" and move the file into position atomically
143 # Our lock prevents multiple writers but mirroring code may grab incomplete files
144 os.rename(localpath, localpath[:-4])
145
126 return True 146 return True
127 147
128 def checkstatus(self, fetch, ud, d, try_again=True): 148 def checkstatus(self, fetch, ud, d, try_again=True):
@@ -209,12 +229,17 @@ class Wget(FetchMethod):
209 # We let the request fail and expect it to be 229 # We let the request fail and expect it to be
210 # tried once more ("try_again" in check_status()), 230 # tried once more ("try_again" in check_status()),
211 # with the dead connection removed from the cache. 231 # with the dead connection removed from the cache.
212 # If it still fails, we give up, which can happend for bad 232 # If it still fails, we give up, which can happen for bad
213 # HTTP proxy settings. 233 # HTTP proxy settings.
214 fetch.connection_cache.remove_connection(h.host, h.port) 234 fetch.connection_cache.remove_connection(h.host, h.port)
215 raise urllib.error.URLError(err) 235 raise urllib.error.URLError(err)
216 else: 236 else:
217 r = h.getresponse() 237 try:
238 r = h.getresponse()
239 except TimeoutError as e:
240 if fetch.connection_cache:
241 fetch.connection_cache.remove_connection(h.host, h.port)
242 raise TimeoutError(e)
218 243
219 # Pick apart the HTTPResponse object to get the addinfourl 244 # Pick apart the HTTPResponse object to get the addinfourl
220 # object initialized properly. 245 # object initialized properly.
@@ -275,71 +300,115 @@ class Wget(FetchMethod):
275 300
276 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): 301 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
277 """ 302 """
278 urllib2.HTTPRedirectHandler resets the method to GET on redirect, 303 urllib2.HTTPRedirectHandler before 3.13 has two flaws:
279 when we want to follow redirects using the original method. 304
305 It resets the method to GET on redirect when we want to follow
306 redirects using the original method (typically HEAD). This was fixed
307 in 759e8e7.
308
309 It also doesn't handle 308 (Permanent Redirect). This was fixed in
310 c379bc5.
311
312 Until we depend on Python 3.13 onwards, copy the redirect_request
313 method to fix these issues.
280 """ 314 """
281 def redirect_request(self, req, fp, code, msg, headers, newurl): 315 def redirect_request(self, req, fp, code, msg, headers, newurl):
282 newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) 316 m = req.get_method()
283 newreq.get_method = req.get_method 317 if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
284 return newreq 318 or code in (301, 302, 303) and m == "POST")):
285 exported_proxies = export_proxies(d) 319 raise urllib.HTTPError(req.full_url, code, msg, headers, fp)
286 320
287 handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] 321 # Strictly (according to RFC 2616), 301 or 302 in response to
288 if exported_proxies: 322 # a POST MUST NOT cause a redirection without confirmation
289 handlers.append(urllib.request.ProxyHandler()) 323 # from the user (of urllib.request, in this case). In practice,
290 handlers.append(CacheHTTPHandler()) 324 # essentially all clients do redirect in this case, so we do
291 # Since Python 2.7.9 ssl cert validation is enabled by default 325 # the same.
292 # see PEP-0476, this causes verification errors on some https servers 326
293 # so disable by default. 327 # Be conciliant with URIs containing a space. This is mainly
294 import ssl 328 # redundant with the more complete encoding done in http_error_302(),
295 if hasattr(ssl, '_create_unverified_context'): 329 # but it is kept for compatibility with other callers.
296 handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) 330 newurl = newurl.replace(' ', '%20')
297 opener = urllib.request.build_opener(*handlers) 331
298 332 CONTENT_HEADERS = ("content-length", "content-type")
299 try: 333 newheaders = {k: v for k, v in req.headers.items()
300 uri = ud.url.split(";")[0] 334 if k.lower() not in CONTENT_HEADERS}
301 r = urllib.request.Request(uri) 335 return urllib.request.Request(newurl,
302 r.get_method = lambda: "HEAD" 336 method="HEAD" if m == "HEAD" else "GET",
303 # Some servers (FusionForge, as used on Alioth) require that the 337 headers=newheaders,
304 # optional Accept header is set. 338 origin_req_host=req.origin_req_host,
305 r.add_header("Accept", "*/*") 339 unverifiable=True)
306 r.add_header("User-Agent", self.user_agent) 340
307 def add_basic_auth(login_str, request): 341 http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
308 '''Adds Basic auth to http request, pass in login:password as string''' 342
309 import base64 343 # We need to update the environment here as both the proxy and HTTPS
310 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") 344 # handlers need variables set. The proxy needs http_proxy and friends to
311 authheader = "Basic %s" % encodeuser 345 # be set, and HTTPSHandler ends up calling into openssl to load the
312 r.add_header("Authorization", authheader) 346 # certificates. In buildtools configurations this will be looking at the
313 347 # wrong place for certificates by default: we set SSL_CERT_FILE to the
314 if ud.user and ud.pswd: 348 # right location in the buildtools environment script but as BitBake
315 add_basic_auth(ud.user + ':' + ud.pswd, r) 349 # prunes prunes the environment this is lost. When binaries are executed
350 # runfetchcmd ensures these values are in the environment, but this is
351 # pure Python so we need to update the environment.
352 #
353 # Avoid tramping the environment too much by using bb.utils.environment
354 # to scope the changes to the build_opener request, which is when the
355 # environment lookups happen.
356 newenv = bb.fetch2.get_fetcher_environment(d)
357
358 with bb.utils.environment(**newenv):
359 import ssl
360
361 if self.check_certs(d):
362 context = ssl.create_default_context()
363 else:
364 context = ssl._create_unverified_context()
365
366 handlers = [FixedHTTPRedirectHandler,
367 HTTPMethodFallback,
368 urllib.request.ProxyHandler(),
369 CacheHTTPHandler(),
370 urllib.request.HTTPSHandler(context=context)]
371 opener = urllib.request.build_opener(*handlers)
316 372
317 try: 373 try:
318 import netrc 374 parts = urllib.parse.urlparse(ud.url.split(";")[0])
319 n = netrc.netrc() 375 uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path)
320 login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) 376 r = urllib.request.Request(uri)
321 add_basic_auth("%s:%s" % (login, password), r) 377 r.get_method = lambda: "HEAD"
322 except (TypeError, ImportError, IOError, netrc.NetrcParseError): 378 # Some servers (FusionForge, as used on Alioth) require that the
323 pass 379 # optional Accept header is set.
324 380 r.add_header("Accept", "*/*")
325 with opener.open(r) as response: 381 r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
326 pass 382 def add_basic_auth(login_str, request):
327 except urllib.error.URLError as e: 383 '''Adds Basic auth to http request, pass in login:password as string'''
328 if try_again: 384 import base64
329 logger.debug2("checkstatus: trying again") 385 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
330 return self.checkstatus(fetch, ud, d, False) 386 authheader = "Basic %s" % encodeuser
331 else: 387 r.add_header("Authorization", authheader)
332 # debug for now to avoid spamming the logs in e.g. remote sstate searches 388
333 logger.debug2("checkstatus() urlopen failed: %s" % e) 389 if ud.user and ud.pswd:
334 return False 390 add_basic_auth(ud.user + ':' + ud.pswd, r)
335 except ConnectionResetError as e: 391
336 if try_again: 392 try:
337 logger.debug2("checkstatus: trying again") 393 import netrc
338 return self.checkstatus(fetch, ud, d, False) 394 auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
339 else: 395 if auth_data:
340 # debug for now to avoid spamming the logs in e.g. remote sstate searches 396 login, _, password = auth_data
341 logger.debug2("checkstatus() urlopen failed: %s" % e) 397 add_basic_auth("%s:%s" % (login, password), r)
342 return False 398 except (FileNotFoundError, netrc.NetrcParseError):
399 pass
400
401 with opener.open(r, timeout=100) as response:
402 pass
403 except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
404 if try_again:
405 logger.debug2("checkstatus: trying again")
406 return self.checkstatus(fetch, ud, d, False)
407 else:
408 # debug for now to avoid spamming the logs in e.g. remote sstate searches
409 logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
410 return False
411
343 return True 412 return True
344 413
345 def _parse_path(self, regex, s): 414 def _parse_path(self, regex, s):
@@ -416,7 +485,7 @@ class Wget(FetchMethod):
416 f = tempfile.NamedTemporaryFile() 485 f = tempfile.NamedTemporaryFile()
417 with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: 486 with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
418 fetchcmd = self.basecmd 487 fetchcmd = self.basecmd
419 fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'" 488 fetchcmd += " --output-document=%s '%s'" % (f.name, uri)
420 try: 489 try:
421 self._runwget(ud, d, fetchcmd, True, workdir=workdir) 490 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
422 fetchresult = f.read() 491 fetchresult = f.read()
@@ -472,7 +541,7 @@ class Wget(FetchMethod):
472 version_dir = ['', '', ''] 541 version_dir = ['', '', '']
473 version = ['', '', ''] 542 version = ['', '', '']
474 543
475 dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") 544 dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
476 s = dirver_regex.search(dirver) 545 s = dirver_regex.search(dirver)
477 if s: 546 if s:
478 version_dir[1] = s.group('ver') 547 version_dir[1] = s.group('ver')
@@ -548,7 +617,7 @@ class Wget(FetchMethod):
548 617
549 # src.rpm extension was added only for rpm package. Can be removed if the rpm 618 # src.rpm extension was added only for rpm package. Can be removed if the rpm
550 # packaged will always be considered as having to be manually upgraded 619 # packaged will always be considered as having to be manually upgraded
551 psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" 620 psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
552 621
553 # match name, version and archive type of a package 622 # match name, version and archive type of a package
554 package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" 623 package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
@@ -576,13 +645,17 @@ class Wget(FetchMethod):
576 645
577 sanity check to ensure same name and type. 646 sanity check to ensure same name and type.
578 """ 647 """
579 package = ud.path.split("/")[-1] 648 if 'downloadfilename' in ud.parm:
649 package = ud.parm['downloadfilename']
650 else:
651 package = ud.path.split("/")[-1]
580 current_version = ['', d.getVar('PV'), ''] 652 current_version = ['', d.getVar('PV'), '']
581 653
582 """possible to have no version in pkg name, such as spectrum-fw""" 654 """possible to have no version in pkg name, such as spectrum-fw"""
583 if not re.search(r"\d+", package): 655 if not re.search(r"\d+", package):
584 current_version[1] = re.sub('_', '.', current_version[1]) 656 current_version[1] = re.sub('_', '.', current_version[1])
585 current_version[1] = re.sub('-', '.', current_version[1]) 657 current_version[1] = re.sub('-', '.', current_version[1])
658 bb.debug(3, "latest_versionstring: no version found in %s" % package)
586 return (current_version[1], '') 659 return (current_version[1], '')
587 660
588 package_regex = self._init_regexes(package, ud, d) 661 package_regex = self._init_regexes(package, ud, d)
@@ -599,10 +672,10 @@ class Wget(FetchMethod):
599 # search for version matches on folders inside the path, like: 672 # search for version matches on folders inside the path, like:
600 # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz 673 # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
601 dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") 674 dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
602 m = dirver_regex.search(path) 675 m = dirver_regex.findall(path)
603 if m: 676 if m:
604 pn = d.getVar('PN') 677 pn = d.getVar('PN')
605 dirver = m.group('dirver') 678 dirver = m[-1][0]
606 679
607 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) 680 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
608 if not dirver_pn_regex.search(dirver): 681 if not dirver_pn_regex.search(dirver):