diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r-- | bitbake/lib/bb/fetch2/README | 57 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 365 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/az.py | 93 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/crate.py | 150 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gcp.py | 102 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 294 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gitsm.py | 49 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/hg.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/local.py | 16 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npm.py | 63 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npmsw.py | 96 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/osc.py | 52 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/perforce.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/s3.py | 41 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/sftp.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/ssh.py | 47 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/svn.py | 15 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/wget.py | 189 |
18 files changed, 1309 insertions, 325 deletions
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README new file mode 100644 index 0000000000..67b787ef47 --- /dev/null +++ b/bitbake/lib/bb/fetch2/README | |||
@@ -0,0 +1,57 @@ | |||
1 | There are expectations of users of the fetcher code. This file attempts to document | ||
2 | some of the constraints that are present. Some are obvious, some are less so. It is | ||
3 | documented in the context of how OE uses it but the API calls are generic. | ||
4 | |||
5 | a) network access for sources is only expected to happen in the do_fetch task. | ||
6 | This is not enforced or tested but is required so that we can: | ||
7 | |||
8 | i) audit the sources used (i.e. for license/manifest reasons) | ||
9 | ii) support offline builds with a suitable cache | ||
10 | iii) allow work to continue even with downtime upstream | ||
11 | iv) allow for changes upstream in incompatible ways | ||
12 | v) allow rebuilding of the software in X years time | ||
13 | |||
14 | b) network access is not expected in do_unpack task. | ||
15 | |||
16 | c) you can take DL_DIR and use it as a mirror for offline builds. | ||
17 | |||
18 | d) access to the network is only made when explicitly configured in recipes | ||
19 | (e.g. use of AUTOREV, or use of git tags which change revision). | ||
20 | |||
21 | e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it | ||
22 | will match in future exactly in a clean build with a new DL_DIR). | ||
23 | One specific pain point example are git tags. They can be replaced and change | ||
24 | so the git fetcher has to resolve them with the network. We use git revisions | ||
25 | where possible to avoid this and ensure determinism. | ||
26 | |||
27 | f) network access is expected to work with the standard linux proxy variables | ||
28 | so that access behind firewalls works (the fetcher sets these in the | ||
29 | environment but only in the do_fetch tasks). | ||
30 | |||
31 | g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV | ||
32 | git recipe might be ok but you can't expect to checkout a git tree. | ||
33 | |||
34 | h) we need to provide revision information during parsing such that a version | ||
35 | for the recipe can be constructed. | ||
36 | |||
37 | i) versions are expected to be able to increase in a way which sorts allowing | ||
38 | package feeds to operate (see PR server required for git revisions to sort). | ||
39 | |||
40 | j) API to query for possible version upgrades of a url is highly desireable to | ||
41 | allow our automated upgrage code to function (it is implied this does always | ||
42 | have network access). | ||
43 | |||
44 | k) Where fixes or changes to behaviour in the fetcher are made, we ask that | ||
45 | test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do | ||
46 | have fairly extensive test coverage of the fetcher as it is the only way | ||
47 | to track all of its corner cases, it still doesn't give entire coverage | ||
48 | though sadly. | ||
49 | |||
50 | l) If using tools during parse time, they will have to be in ASSUME_PROVIDED | ||
51 | in OE's context as we can't build git-native, then parse a recipe and use | ||
52 | git ls-remote. | ||
53 | |||
54 | Not all fetchers support all features, autorev is optional and doesn't make | ||
55 | sense for some. Upgrade detection means different things in different contexts | ||
56 | too. | ||
57 | |||
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 19169d780f..5bf2c4b8cf 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py | |||
@@ -113,7 +113,7 @@ class MissingParameterError(BBFetchException): | |||
113 | self.args = (missing, url) | 113 | self.args = (missing, url) |
114 | 114 | ||
115 | class ParameterError(BBFetchException): | 115 | class ParameterError(BBFetchException): |
116 | """Exception raised when a url cannot be proccessed due to invalid parameters.""" | 116 | """Exception raised when a url cannot be processed due to invalid parameters.""" |
117 | def __init__(self, message, url): | 117 | def __init__(self, message, url): |
118 | msg = "URL: '%s' has invalid parameters. %s" % (url, message) | 118 | msg = "URL: '%s' has invalid parameters. %s" % (url, message) |
119 | self.url = url | 119 | self.url = url |
@@ -182,7 +182,7 @@ class URI(object): | |||
182 | Some notes about relative URIs: while it's specified that | 182 | Some notes about relative URIs: while it's specified that |
183 | a URI beginning with <scheme>:// should either be directly | 183 | a URI beginning with <scheme>:// should either be directly |
184 | followed by a hostname or a /, the old URI handling of the | 184 | followed by a hostname or a /, the old URI handling of the |
185 | fetch2 library did not comform to this. Therefore, this URI | 185 | fetch2 library did not conform to this. Therefore, this URI |
186 | class has some kludges to make sure that URIs are parsed in | 186 | class has some kludges to make sure that URIs are parsed in |
187 | a way comforming to bitbake's current usage. This URI class | 187 | a way comforming to bitbake's current usage. This URI class |
188 | supports the following: | 188 | supports the following: |
@@ -199,7 +199,7 @@ class URI(object): | |||
199 | file://hostname/absolute/path.diff (would be IETF compliant) | 199 | file://hostname/absolute/path.diff (would be IETF compliant) |
200 | 200 | ||
201 | Note that the last case only applies to a list of | 201 | Note that the last case only applies to a list of |
202 | "whitelisted" schemes (currently only file://), that requires | 202 | explicitly allowed schemes (currently only file://), that requires |
203 | its URIs to not have a network location. | 203 | its URIs to not have a network location. |
204 | """ | 204 | """ |
205 | 205 | ||
@@ -290,12 +290,12 @@ class URI(object): | |||
290 | 290 | ||
291 | def _param_str_split(self, string, elmdelim, kvdelim="="): | 291 | def _param_str_split(self, string, elmdelim, kvdelim="="): |
292 | ret = collections.OrderedDict() | 292 | ret = collections.OrderedDict() |
293 | for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: | 293 | for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]: |
294 | ret[k] = v | 294 | ret[k] = v |
295 | return ret | 295 | return ret |
296 | 296 | ||
297 | def _param_str_join(self, dict_, elmdelim, kvdelim="="): | 297 | def _param_str_join(self, dict_, elmdelim, kvdelim="="): |
298 | return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) | 298 | return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()]) |
299 | 299 | ||
300 | @property | 300 | @property |
301 | def hostport(self): | 301 | def hostport(self): |
@@ -388,7 +388,7 @@ def decodeurl(url): | |||
388 | if s: | 388 | if s: |
389 | if not '=' in s: | 389 | if not '=' in s: |
390 | raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) | 390 | raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) |
391 | s1, s2 = s.split('=') | 391 | s1, s2 = s.split('=', 1) |
392 | p[s1] = s2 | 392 | p[s1] = s2 |
393 | 393 | ||
394 | return type, host, urllib.parse.unquote(path), user, pswd, p | 394 | return type, host, urllib.parse.unquote(path), user, pswd, p |
@@ -402,24 +402,24 @@ def encodeurl(decoded): | |||
402 | 402 | ||
403 | if not type: | 403 | if not type: |
404 | raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) | 404 | raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) |
405 | url = '%s://' % type | 405 | url = ['%s://' % type] |
406 | if user and type != "file": | 406 | if user and type != "file": |
407 | url += "%s" % user | 407 | url.append("%s" % user) |
408 | if pswd: | 408 | if pswd: |
409 | url += ":%s" % pswd | 409 | url.append(":%s" % pswd) |
410 | url += "@" | 410 | url.append("@") |
411 | if host and type != "file": | 411 | if host and type != "file": |
412 | url += "%s" % host | 412 | url.append("%s" % host) |
413 | if path: | 413 | if path: |
414 | # Standardise path to ensure comparisons work | 414 | # Standardise path to ensure comparisons work |
415 | while '//' in path: | 415 | while '//' in path: |
416 | path = path.replace("//", "/") | 416 | path = path.replace("//", "/") |
417 | url += "%s" % urllib.parse.quote(path) | 417 | url.append("%s" % urllib.parse.quote(path)) |
418 | if p: | 418 | if p: |
419 | for parm in p: | 419 | for parm in p: |
420 | url += ";%s=%s" % (parm, p[parm]) | 420 | url.append(";%s=%s" % (parm, p[parm])) |
421 | 421 | ||
422 | return url | 422 | return "".join(url) |
423 | 423 | ||
424 | def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | 424 | def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): |
425 | if not ud.url or not uri_find or not uri_replace: | 425 | if not ud.url or not uri_find or not uri_replace: |
@@ -430,6 +430,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
430 | uri_replace_decoded = list(decodeurl(uri_replace)) | 430 | uri_replace_decoded = list(decodeurl(uri_replace)) |
431 | logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) | 431 | logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) |
432 | result_decoded = ['', '', '', '', '', {}] | 432 | result_decoded = ['', '', '', '', '', {}] |
433 | # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params | ||
433 | for loc, i in enumerate(uri_find_decoded): | 434 | for loc, i in enumerate(uri_find_decoded): |
434 | result_decoded[loc] = uri_decoded[loc] | 435 | result_decoded[loc] = uri_decoded[loc] |
435 | regexp = i | 436 | regexp = i |
@@ -449,6 +450,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
449 | for l in replacements: | 450 | for l in replacements: |
450 | uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) | 451 | uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) |
451 | result_decoded[loc][k] = uri_replace_decoded[loc][k] | 452 | result_decoded[loc][k] = uri_replace_decoded[loc][k] |
453 | elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]: | ||
454 | # User/password in the replacement is just a straight replacement | ||
455 | result_decoded[loc] = uri_replace_decoded[loc] | ||
452 | elif (re.match(regexp, uri_decoded[loc])): | 456 | elif (re.match(regexp, uri_decoded[loc])): |
453 | if not uri_replace_decoded[loc]: | 457 | if not uri_replace_decoded[loc]: |
454 | result_decoded[loc] = "" | 458 | result_decoded[loc] = "" |
@@ -465,10 +469,18 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
465 | basename = os.path.basename(mirrortarball) | 469 | basename = os.path.basename(mirrortarball) |
466 | # Kill parameters, they make no sense for mirror tarballs | 470 | # Kill parameters, they make no sense for mirror tarballs |
467 | uri_decoded[5] = {} | 471 | uri_decoded[5] = {} |
472 | uri_find_decoded[5] = {} | ||
468 | elif ud.localpath and ud.method.supports_checksum(ud): | 473 | elif ud.localpath and ud.method.supports_checksum(ud): |
469 | basename = os.path.basename(ud.localpath) | 474 | basename = os.path.basename(ud.localpath) |
470 | if basename and not result_decoded[loc].endswith(basename): | 475 | if basename: |
471 | result_decoded[loc] = os.path.join(result_decoded[loc], basename) | 476 | uri_basename = os.path.basename(uri_decoded[loc]) |
477 | # Prefix with a slash as a sentinel in case | ||
478 | # result_decoded[loc] does not contain one. | ||
479 | path = "/" + result_decoded[loc] | ||
480 | if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): | ||
481 | result_decoded[loc] = path[1:-len(uri_basename)] + basename | ||
482 | elif not path.endswith("/" + basename): | ||
483 | result_decoded[loc] = os.path.join(path[1:], basename) | ||
472 | else: | 484 | else: |
473 | return None | 485 | return None |
474 | result = encodeurl(result_decoded) | 486 | result = encodeurl(result_decoded) |
@@ -506,7 +518,7 @@ def fetcher_init(d): | |||
506 | else: | 518 | else: |
507 | raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) | 519 | raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) |
508 | 520 | ||
509 | _checksum_cache.init_cache(d) | 521 | _checksum_cache.init_cache(d.getVar("BB_CACHEDIR")) |
510 | 522 | ||
511 | for m in methods: | 523 | for m in methods: |
512 | if hasattr(m, "init"): | 524 | if hasattr(m, "init"): |
@@ -534,7 +546,7 @@ def mirror_from_string(data): | |||
534 | bb.warn('Invalid mirror data %s, should have paired members.' % data) | 546 | bb.warn('Invalid mirror data %s, should have paired members.' % data) |
535 | return list(zip(*[iter(mirrors)]*2)) | 547 | return list(zip(*[iter(mirrors)]*2)) |
536 | 548 | ||
537 | def verify_checksum(ud, d, precomputed={}): | 549 | def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True): |
538 | """ | 550 | """ |
539 | verify the MD5 and SHA256 checksum for downloaded src | 551 | verify the MD5 and SHA256 checksum for downloaded src |
540 | 552 | ||
@@ -548,20 +560,25 @@ def verify_checksum(ud, d, precomputed={}): | |||
548 | file against those in the recipe each time, rather than only after | 560 | file against those in the recipe each time, rather than only after |
549 | downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. | 561 | downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. |
550 | """ | 562 | """ |
551 | |||
552 | if ud.ignore_checksums or not ud.method.supports_checksum(ud): | 563 | if ud.ignore_checksums or not ud.method.supports_checksum(ud): |
553 | return {} | 564 | return {} |
554 | 565 | ||
566 | if localpath is None: | ||
567 | localpath = ud.localpath | ||
568 | |||
555 | def compute_checksum_info(checksum_id): | 569 | def compute_checksum_info(checksum_id): |
556 | checksum_name = getattr(ud, "%s_name" % checksum_id) | 570 | checksum_name = getattr(ud, "%s_name" % checksum_id) |
557 | 571 | ||
558 | if checksum_id in precomputed: | 572 | if checksum_id in precomputed: |
559 | checksum_data = precomputed[checksum_id] | 573 | checksum_data = precomputed[checksum_id] |
560 | else: | 574 | else: |
561 | checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) | 575 | checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath) |
562 | 576 | ||
563 | checksum_expected = getattr(ud, "%s_expected" % checksum_id) | 577 | checksum_expected = getattr(ud, "%s_expected" % checksum_id) |
564 | 578 | ||
579 | if checksum_expected == '': | ||
580 | checksum_expected = None | ||
581 | |||
565 | return { | 582 | return { |
566 | "id": checksum_id, | 583 | "id": checksum_id, |
567 | "name": checksum_name, | 584 | "name": checksum_name, |
@@ -581,17 +598,13 @@ def verify_checksum(ud, d, precomputed={}): | |||
581 | checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] | 598 | checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] |
582 | 599 | ||
583 | # If no checksum has been provided | 600 | # If no checksum has been provided |
584 | if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): | 601 | if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): |
585 | messages = [] | 602 | messages = [] |
586 | strict = d.getVar("BB_STRICT_CHECKSUM") or "0" | 603 | strict = d.getVar("BB_STRICT_CHECKSUM") or "0" |
587 | 604 | ||
588 | # If strict checking enabled and neither sum defined, raise error | 605 | # If strict checking enabled and neither sum defined, raise error |
589 | if strict == "1": | 606 | if strict == "1": |
590 | messages.append("No checksum specified for '%s', please add at " \ | 607 | raise NoChecksumError("\n".join(checksum_lines)) |
591 | "least one to the recipe:" % ud.localpath) | ||
592 | messages.extend(checksum_lines) | ||
593 | logger.error("\n".join(messages)) | ||
594 | raise NoChecksumError("Missing SRC_URI checksum", ud.url) | ||
595 | 608 | ||
596 | bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) | 609 | bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) |
597 | 610 | ||
@@ -612,8 +625,8 @@ def verify_checksum(ud, d, precomputed={}): | |||
612 | 625 | ||
613 | for ci in checksum_infos: | 626 | for ci in checksum_infos: |
614 | if ci["expected"] and ci["expected"] != ci["data"]: | 627 | if ci["expected"] and ci["expected"] != ci["data"]: |
615 | messages.append("File: '%s' has %s checksum %s when %s was " \ | 628 | messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ |
616 | "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) | 629 | "expected" % (localpath, ci["id"], ci["data"], ci["expected"])) |
617 | bad_checksum = ci["data"] | 630 | bad_checksum = ci["data"] |
618 | 631 | ||
619 | if bad_checksum: | 632 | if bad_checksum: |
@@ -731,13 +744,16 @@ def subprocess_setup(): | |||
731 | # SIGPIPE errors are known issues with gzip/bash | 744 | # SIGPIPE errors are known issues with gzip/bash |
732 | signal.signal(signal.SIGPIPE, signal.SIG_DFL) | 745 | signal.signal(signal.SIGPIPE, signal.SIG_DFL) |
733 | 746 | ||
734 | def get_autorev(d): | 747 | def mark_recipe_nocache(d): |
735 | # only not cache src rev in autorev case | ||
736 | if d.getVar('BB_SRCREV_POLICY') != "cache": | 748 | if d.getVar('BB_SRCREV_POLICY') != "cache": |
737 | d.setVar('BB_DONT_CACHE', '1') | 749 | d.setVar('BB_DONT_CACHE', '1') |
750 | |||
751 | def get_autorev(d): | ||
752 | mark_recipe_nocache(d) | ||
753 | d.setVar("__BBAUTOREV_SEEN", True) | ||
738 | return "AUTOINC" | 754 | return "AUTOINC" |
739 | 755 | ||
740 | def get_srcrev(d, method_name='sortable_revision'): | 756 | def _get_srcrev(d, method_name='sortable_revision'): |
741 | """ | 757 | """ |
742 | Return the revision string, usually for use in the version string (PV) of the current package | 758 | Return the revision string, usually for use in the version string (PV) of the current package |
743 | Most packages usually only have one SCM so we just pass on the call. | 759 | Most packages usually only have one SCM so we just pass on the call. |
@@ -751,23 +767,34 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
751 | that fetcher provides a method with the given name and the same signature as sortable_revision. | 767 | that fetcher provides a method with the given name and the same signature as sortable_revision. |
752 | """ | 768 | """ |
753 | 769 | ||
770 | d.setVar("__BBSRCREV_SEEN", "1") | ||
771 | recursion = d.getVar("__BBINSRCREV") | ||
772 | if recursion: | ||
773 | raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") | ||
774 | d.setVar("__BBINSRCREV", True) | ||
775 | |||
754 | scms = [] | 776 | scms = [] |
777 | revs = [] | ||
755 | fetcher = Fetch(d.getVar('SRC_URI').split(), d) | 778 | fetcher = Fetch(d.getVar('SRC_URI').split(), d) |
756 | urldata = fetcher.ud | 779 | urldata = fetcher.ud |
757 | for u in urldata: | 780 | for u in urldata: |
758 | if urldata[u].method.supports_srcrev(): | 781 | if urldata[u].method.supports_srcrev(): |
759 | scms.append(u) | 782 | scms.append(u) |
760 | 783 | ||
761 | if len(scms) == 0: | 784 | if not scms: |
762 | raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") | 785 | d.delVar("__BBINSRCREV") |
786 | return "", revs | ||
787 | |||
763 | 788 | ||
764 | if len(scms) == 1 and len(urldata[scms[0]].names) == 1: | 789 | if len(scms) == 1 and len(urldata[scms[0]].names) == 1: |
765 | autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) | 790 | autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) |
791 | revs.append(rev) | ||
766 | if len(rev) > 10: | 792 | if len(rev) > 10: |
767 | rev = rev[:10] | 793 | rev = rev[:10] |
794 | d.delVar("__BBINSRCREV") | ||
768 | if autoinc: | 795 | if autoinc: |
769 | return "AUTOINC+" + rev | 796 | return "AUTOINC+" + rev, revs |
770 | return rev | 797 | return rev, revs |
771 | 798 | ||
772 | # | 799 | # |
773 | # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT | 800 | # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT |
@@ -783,6 +810,7 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
783 | ud = urldata[scm] | 810 | ud = urldata[scm] |
784 | for name in ud.names: | 811 | for name in ud.names: |
785 | autoinc, rev = getattr(ud.method, method_name)(ud, d, name) | 812 | autoinc, rev = getattr(ud.method, method_name)(ud, d, name) |
813 | revs.append(rev) | ||
786 | seenautoinc = seenautoinc or autoinc | 814 | seenautoinc = seenautoinc or autoinc |
787 | if len(rev) > 10: | 815 | if len(rev) > 10: |
788 | rev = rev[:10] | 816 | rev = rev[:10] |
@@ -799,12 +827,70 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
799 | if seenautoinc: | 827 | if seenautoinc: |
800 | format = "AUTOINC+" + format | 828 | format = "AUTOINC+" + format |
801 | 829 | ||
802 | return format | 830 | d.delVar("__BBINSRCREV") |
831 | return format, revs | ||
832 | |||
833 | def get_hashvalue(d, method_name='sortable_revision'): | ||
834 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
835 | return " ".join(revs) | ||
836 | |||
837 | def get_pkgv_string(d, method_name='sortable_revision'): | ||
838 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
839 | return pkgv | ||
840 | |||
841 | def get_srcrev(d, method_name='sortable_revision'): | ||
842 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
843 | if not pkgv: | ||
844 | raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") | ||
845 | return pkgv | ||
803 | 846 | ||
804 | def localpath(url, d): | 847 | def localpath(url, d): |
805 | fetcher = bb.fetch2.Fetch([url], d) | 848 | fetcher = bb.fetch2.Fetch([url], d) |
806 | return fetcher.localpath(url) | 849 | return fetcher.localpath(url) |
807 | 850 | ||
851 | # Need to export PATH as binary could be in metadata paths | ||
852 | # rather than host provided | ||
853 | # Also include some other variables. | ||
854 | FETCH_EXPORT_VARS = ['HOME', 'PATH', | ||
855 | 'HTTP_PROXY', 'http_proxy', | ||
856 | 'HTTPS_PROXY', 'https_proxy', | ||
857 | 'FTP_PROXY', 'ftp_proxy', | ||
858 | 'FTPS_PROXY', 'ftps_proxy', | ||
859 | 'NO_PROXY', 'no_proxy', | ||
860 | 'ALL_PROXY', 'all_proxy', | ||
861 | 'GIT_PROXY_COMMAND', | ||
862 | 'GIT_SSH', | ||
863 | 'GIT_SSH_COMMAND', | ||
864 | 'GIT_SSL_CAINFO', | ||
865 | 'GIT_SMART_HTTP', | ||
866 | 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', | ||
867 | 'SOCKS5_USER', 'SOCKS5_PASSWD', | ||
868 | 'DBUS_SESSION_BUS_ADDRESS', | ||
869 | 'P4CONFIG', | ||
870 | 'SSL_CERT_FILE', | ||
871 | 'NODE_EXTRA_CA_CERTS', | ||
872 | 'AWS_PROFILE', | ||
873 | 'AWS_ACCESS_KEY_ID', | ||
874 | 'AWS_SECRET_ACCESS_KEY', | ||
875 | 'AWS_ROLE_ARN', | ||
876 | 'AWS_WEB_IDENTITY_TOKEN_FILE', | ||
877 | 'AWS_DEFAULT_REGION', | ||
878 | 'AWS_SESSION_TOKEN', | ||
879 | 'GIT_CACHE_PATH', | ||
880 | 'REMOTE_CONTAINERS_IPC', | ||
881 | 'SSL_CERT_DIR'] | ||
882 | |||
883 | def get_fetcher_environment(d): | ||
884 | newenv = {} | ||
885 | origenv = d.getVar("BB_ORIGENV") | ||
886 | for name in bb.fetch2.FETCH_EXPORT_VARS: | ||
887 | value = d.getVar(name) | ||
888 | if not value and origenv: | ||
889 | value = origenv.getVar(name) | ||
890 | if value: | ||
891 | newenv[name] = value | ||
892 | return newenv | ||
893 | |||
808 | def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | 894 | def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): |
809 | """ | 895 | """ |
810 | Run cmd returning the command output | 896 | Run cmd returning the command output |
@@ -813,25 +899,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | |||
813 | Optionally remove the files/directories listed in cleanup upon failure | 899 | Optionally remove the files/directories listed in cleanup upon failure |
814 | """ | 900 | """ |
815 | 901 | ||
816 | # Need to export PATH as binary could be in metadata paths | 902 | exportvars = FETCH_EXPORT_VARS |
817 | # rather than host provided | ||
818 | # Also include some other variables. | ||
819 | # FIXME: Should really include all export varaiables? | ||
820 | exportvars = ['HOME', 'PATH', | ||
821 | 'HTTP_PROXY', 'http_proxy', | ||
822 | 'HTTPS_PROXY', 'https_proxy', | ||
823 | 'FTP_PROXY', 'ftp_proxy', | ||
824 | 'FTPS_PROXY', 'ftps_proxy', | ||
825 | 'NO_PROXY', 'no_proxy', | ||
826 | 'ALL_PROXY', 'all_proxy', | ||
827 | 'GIT_PROXY_COMMAND', | ||
828 | 'GIT_SSH', | ||
829 | 'GIT_SSL_CAINFO', | ||
830 | 'GIT_SMART_HTTP', | ||
831 | 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', | ||
832 | 'SOCKS5_USER', 'SOCKS5_PASSWD', | ||
833 | 'DBUS_SESSION_BUS_ADDRESS', | ||
834 | 'P4CONFIG'] | ||
835 | 903 | ||
836 | if not cleanup: | 904 | if not cleanup: |
837 | cleanup = [] | 905 | cleanup = [] |
@@ -868,14 +936,17 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | |||
868 | (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) | 936 | (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) |
869 | success = True | 937 | success = True |
870 | except bb.process.NotFoundError as e: | 938 | except bb.process.NotFoundError as e: |
871 | error_message = "Fetch command %s" % (e.command) | 939 | error_message = "Fetch command %s not found" % (e.command) |
872 | except bb.process.ExecutionError as e: | 940 | except bb.process.ExecutionError as e: |
873 | if e.stdout: | 941 | if e.stdout: |
874 | output = "output:\n%s\n%s" % (e.stdout, e.stderr) | 942 | output = "output:\n%s\n%s" % (e.stdout, e.stderr) |
875 | elif e.stderr: | 943 | elif e.stderr: |
876 | output = "output:\n%s" % e.stderr | 944 | output = "output:\n%s" % e.stderr |
877 | else: | 945 | else: |
878 | output = "no output" | 946 | if log: |
947 | output = "see logfile for output" | ||
948 | else: | ||
949 | output = "no output" | ||
879 | error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) | 950 | error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) |
880 | except bb.process.CmdError as e: | 951 | except bb.process.CmdError as e: |
881 | error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) | 952 | error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) |
@@ -937,6 +1008,7 @@ def build_mirroruris(origud, mirrors, ld): | |||
937 | 1008 | ||
938 | try: | 1009 | try: |
939 | newud = FetchData(newuri, ld) | 1010 | newud = FetchData(newuri, ld) |
1011 | newud.ignore_checksums = True | ||
940 | newud.setup_localpath(ld) | 1012 | newud.setup_localpath(ld) |
941 | except bb.fetch2.BBFetchException as e: | 1013 | except bb.fetch2.BBFetchException as e: |
942 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) | 1014 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) |
@@ -1046,7 +1118,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1046 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) | 1118 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) |
1047 | logger.debug(str(e)) | 1119 | logger.debug(str(e)) |
1048 | try: | 1120 | try: |
1049 | ud.method.clean(ud, ld) | 1121 | if ud.method.cleanup_upon_failure(): |
1122 | ud.method.clean(ud, ld) | ||
1050 | except UnboundLocalError: | 1123 | except UnboundLocalError: |
1051 | pass | 1124 | pass |
1052 | return False | 1125 | return False |
@@ -1057,6 +1130,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1057 | 1130 | ||
1058 | def ensure_symlink(target, link_name): | 1131 | def ensure_symlink(target, link_name): |
1059 | if not os.path.exists(link_name): | 1132 | if not os.path.exists(link_name): |
1133 | dirname = os.path.dirname(link_name) | ||
1134 | bb.utils.mkdirhier(dirname) | ||
1060 | if os.path.islink(link_name): | 1135 | if os.path.islink(link_name): |
1061 | # Broken symbolic link | 1136 | # Broken symbolic link |
1062 | os.unlink(link_name) | 1137 | os.unlink(link_name) |
@@ -1140,11 +1215,11 @@ def srcrev_internal_helper(ud, d, name): | |||
1140 | pn = d.getVar("PN") | 1215 | pn = d.getVar("PN") |
1141 | attempts = [] | 1216 | attempts = [] |
1142 | if name != '' and pn: | 1217 | if name != '' and pn: |
1143 | attempts.append("SRCREV_%s_pn-%s" % (name, pn)) | 1218 | attempts.append("SRCREV_%s:pn-%s" % (name, pn)) |
1144 | if name != '': | 1219 | if name != '': |
1145 | attempts.append("SRCREV_%s" % name) | 1220 | attempts.append("SRCREV_%s" % name) |
1146 | if pn: | 1221 | if pn: |
1147 | attempts.append("SRCREV_pn-%s" % pn) | 1222 | attempts.append("SRCREV:pn-%s" % pn) |
1148 | attempts.append("SRCREV") | 1223 | attempts.append("SRCREV") |
1149 | 1224 | ||
1150 | for a in attempts: | 1225 | for a in attempts: |
@@ -1169,6 +1244,7 @@ def srcrev_internal_helper(ud, d, name): | |||
1169 | if srcrev == "INVALID" or not srcrev: | 1244 | if srcrev == "INVALID" or not srcrev: |
1170 | raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) | 1245 | raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) |
1171 | if srcrev == "AUTOINC": | 1246 | if srcrev == "AUTOINC": |
1247 | d.setVar("__BBAUTOREV_ACTED_UPON", True) | ||
1172 | srcrev = ud.method.latest_revision(ud, d, name) | 1248 | srcrev = ud.method.latest_revision(ud, d, name) |
1173 | 1249 | ||
1174 | return srcrev | 1250 | return srcrev |
@@ -1180,23 +1256,21 @@ def get_checksum_file_list(d): | |||
1180 | SRC_URI as a space-separated string | 1256 | SRC_URI as a space-separated string |
1181 | """ | 1257 | """ |
1182 | fetch = Fetch([], d, cache = False, localonly = True) | 1258 | fetch = Fetch([], d, cache = False, localonly = True) |
1183 | |||
1184 | dl_dir = d.getVar('DL_DIR') | ||
1185 | filelist = [] | 1259 | filelist = [] |
1186 | for u in fetch.urls: | 1260 | for u in fetch.urls: |
1187 | ud = fetch.ud[u] | 1261 | ud = fetch.ud[u] |
1188 | |||
1189 | if ud and isinstance(ud.method, local.Local): | 1262 | if ud and isinstance(ud.method, local.Local): |
1190 | paths = ud.method.localpaths(ud, d) | 1263 | found = False |
1264 | paths = ud.method.localfile_searchpaths(ud, d) | ||
1191 | for f in paths: | 1265 | for f in paths: |
1192 | pth = ud.decodedurl | 1266 | pth = ud.decodedurl |
1193 | if f.startswith(dl_dir): | 1267 | if os.path.exists(f): |
1194 | # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else | 1268 | found = True |
1195 | if os.path.exists(f): | ||
1196 | bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) | ||
1197 | else: | ||
1198 | bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f))) | ||
1199 | filelist.append(f + ":" + str(os.path.exists(f))) | 1269 | filelist.append(f + ":" + str(os.path.exists(f))) |
1270 | if not found: | ||
1271 | bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found" | ||
1272 | "\nThe following paths were searched:" | ||
1273 | "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths))) | ||
1200 | 1274 | ||
1201 | return " ".join(filelist) | 1275 | return " ".join(filelist) |
1202 | 1276 | ||
@@ -1243,18 +1317,13 @@ class FetchData(object): | |||
1243 | 1317 | ||
1244 | if checksum_name in self.parm: | 1318 | if checksum_name in self.parm: |
1245 | checksum_expected = self.parm[checksum_name] | 1319 | checksum_expected = self.parm[checksum_name] |
1246 | elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: | 1320 | elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]: |
1247 | checksum_expected = None | 1321 | checksum_expected = None |
1248 | else: | 1322 | else: |
1249 | checksum_expected = d.getVarFlag("SRC_URI", checksum_name) | 1323 | checksum_expected = d.getVarFlag("SRC_URI", checksum_name) |
1250 | 1324 | ||
1251 | setattr(self, "%s_expected" % checksum_id, checksum_expected) | 1325 | setattr(self, "%s_expected" % checksum_id, checksum_expected) |
1252 | 1326 | ||
1253 | for checksum_id in CHECKSUM_LIST: | ||
1254 | configure_checksum(checksum_id) | ||
1255 | |||
1256 | self.ignore_checksums = False | ||
1257 | |||
1258 | self.names = self.parm.get("name",'default').split(',') | 1327 | self.names = self.parm.get("name",'default').split(',') |
1259 | 1328 | ||
1260 | self.method = None | 1329 | self.method = None |
@@ -1276,6 +1345,11 @@ class FetchData(object): | |||
1276 | if hasattr(self.method, "urldata_init"): | 1345 | if hasattr(self.method, "urldata_init"): |
1277 | self.method.urldata_init(self, d) | 1346 | self.method.urldata_init(self, d) |
1278 | 1347 | ||
1348 | for checksum_id in CHECKSUM_LIST: | ||
1349 | configure_checksum(checksum_id) | ||
1350 | |||
1351 | self.ignore_checksums = False | ||
1352 | |||
1279 | if "localpath" in self.parm: | 1353 | if "localpath" in self.parm: |
1280 | # if user sets localpath for file, use it instead. | 1354 | # if user sets localpath for file, use it instead. |
1281 | self.localpath = self.parm["localpath"] | 1355 | self.localpath = self.parm["localpath"] |
@@ -1355,6 +1429,9 @@ class FetchMethod(object): | |||
1355 | Is localpath something that can be represented by a checksum? | 1429 | Is localpath something that can be represented by a checksum? |
1356 | """ | 1430 | """ |
1357 | 1431 | ||
1432 | # We cannot compute checksums for None | ||
1433 | if urldata.localpath is None: | ||
1434 | return False | ||
1358 | # We cannot compute checksums for directories | 1435 | # We cannot compute checksums for directories |
1359 | if os.path.isdir(urldata.localpath): | 1436 | if os.path.isdir(urldata.localpath): |
1360 | return False | 1437 | return False |
@@ -1367,6 +1444,12 @@ class FetchMethod(object): | |||
1367 | """ | 1444 | """ |
1368 | return False | 1445 | return False |
1369 | 1446 | ||
1447 | def cleanup_upon_failure(self): | ||
1448 | """ | ||
1449 | When a fetch fails, should clean() be called? | ||
1450 | """ | ||
1451 | return True | ||
1452 | |||
1370 | def verify_donestamp(self, ud, d): | 1453 | def verify_donestamp(self, ud, d): |
1371 | """ | 1454 | """ |
1372 | Verify the donestamp file | 1455 | Verify the donestamp file |
@@ -1434,30 +1517,33 @@ class FetchMethod(object): | |||
1434 | cmd = None | 1517 | cmd = None |
1435 | 1518 | ||
1436 | if unpack: | 1519 | if unpack: |
1520 | tar_cmd = 'tar --extract --no-same-owner' | ||
1521 | if 'striplevel' in urldata.parm: | ||
1522 | tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel'] | ||
1437 | if file.endswith('.tar'): | 1523 | if file.endswith('.tar'): |
1438 | cmd = 'tar x --no-same-owner -f %s' % file | 1524 | cmd = '%s -f %s' % (tar_cmd, file) |
1439 | elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): | 1525 | elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): |
1440 | cmd = 'tar xz --no-same-owner -f %s' % file | 1526 | cmd = '%s -z -f %s' % (tar_cmd, file) |
1441 | elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): | 1527 | elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): |
1442 | cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file | 1528 | cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd) |
1443 | elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): | 1529 | elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): |
1444 | cmd = 'gzip -dc %s > %s' % (file, efile) | 1530 | cmd = 'gzip -dc %s > %s' % (file, efile) |
1445 | elif file.endswith('.bz2'): | 1531 | elif file.endswith('.bz2'): |
1446 | cmd = 'bzip2 -dc %s > %s' % (file, efile) | 1532 | cmd = 'bzip2 -dc %s > %s' % (file, efile) |
1447 | elif file.endswith('.txz') or file.endswith('.tar.xz'): | 1533 | elif file.endswith('.txz') or file.endswith('.tar.xz'): |
1448 | cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file | 1534 | cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd) |
1449 | elif file.endswith('.xz'): | 1535 | elif file.endswith('.xz'): |
1450 | cmd = 'xz -dc %s > %s' % (file, efile) | 1536 | cmd = 'xz -dc %s > %s' % (file, efile) |
1451 | elif file.endswith('.tar.lz'): | 1537 | elif file.endswith('.tar.lz'): |
1452 | cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file | 1538 | cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd) |
1453 | elif file.endswith('.lz'): | 1539 | elif file.endswith('.lz'): |
1454 | cmd = 'lzip -dc %s > %s' % (file, efile) | 1540 | cmd = 'lzip -dc %s > %s' % (file, efile) |
1455 | elif file.endswith('.tar.7z'): | 1541 | elif file.endswith('.tar.7z'): |
1456 | cmd = '7z x -so %s | tar x --no-same-owner -f -' % file | 1542 | cmd = '7z x -so %s | %s -f -' % (file, tar_cmd) |
1457 | elif file.endswith('.7z'): | 1543 | elif file.endswith('.7z'): |
1458 | cmd = '7za x -y %s 1>/dev/null' % file | 1544 | cmd = '7za x -y %s 1>/dev/null' % file |
1459 | elif file.endswith('.tzst') or file.endswith('.tar.zst'): | 1545 | elif file.endswith('.tzst') or file.endswith('.tar.zst'): |
1460 | cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file | 1546 | cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd) |
1461 | elif file.endswith('.zst'): | 1547 | elif file.endswith('.zst'): |
1462 | cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) | 1548 | cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) |
1463 | elif file.endswith('.zip') or file.endswith('.jar'): | 1549 | elif file.endswith('.zip') or file.endswith('.jar'): |
@@ -1490,7 +1576,7 @@ class FetchMethod(object): | |||
1490 | raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) | 1576 | raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) |
1491 | else: | 1577 | else: |
1492 | raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) | 1578 | raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) |
1493 | cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile) | 1579 | cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile) |
1494 | 1580 | ||
1495 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | 1581 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd |
1496 | if 'subdir' in urldata.parm: | 1582 | if 'subdir' in urldata.parm: |
@@ -1506,6 +1592,7 @@ class FetchMethod(object): | |||
1506 | unpackdir = rootdir | 1592 | unpackdir = rootdir |
1507 | 1593 | ||
1508 | if not unpack or not cmd: | 1594 | if not unpack or not cmd: |
1595 | urldata.unpack_tracer.unpack("file-copy", unpackdir) | ||
1509 | # If file == dest, then avoid any copies, as we already put the file into dest! | 1596 | # If file == dest, then avoid any copies, as we already put the file into dest! |
1510 | dest = os.path.join(unpackdir, os.path.basename(file)) | 1597 | dest = os.path.join(unpackdir, os.path.basename(file)) |
1511 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): | 1598 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): |
@@ -1520,6 +1607,8 @@ class FetchMethod(object): | |||
1520 | destdir = urlpath.rsplit("/", 1)[0] + '/' | 1607 | destdir = urlpath.rsplit("/", 1)[0] + '/' |
1521 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) | 1608 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) |
1522 | cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) | 1609 | cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) |
1610 | else: | ||
1611 | urldata.unpack_tracer.unpack("archive-extract", unpackdir) | ||
1523 | 1612 | ||
1524 | if not cmd: | 1613 | if not cmd: |
1525 | return | 1614 | return |
@@ -1611,12 +1700,61 @@ class FetchMethod(object): | |||
1611 | """ | 1700 | """ |
1612 | return [] | 1701 | return [] |
1613 | 1702 | ||
1703 | |||
1704 | class DummyUnpackTracer(object): | ||
1705 | """ | ||
1706 | Abstract API definition for a class that traces unpacked source files back | ||
1707 | to their respective upstream SRC_URI entries, for software composition | ||
1708 | analysis, license compliance and detailed SBOM generation purposes. | ||
1709 | User may load their own unpack tracer class (instead of the dummy | ||
1710 | one) by setting the BB_UNPACK_TRACER_CLASS config parameter. | ||
1711 | """ | ||
1712 | def start(self, unpackdir, urldata_dict, d): | ||
1713 | """ | ||
1714 | Start tracing the core Fetch.unpack process, using an index to map | ||
1715 | unpacked files to each SRC_URI entry. | ||
1716 | This method is called by Fetch.unpack and it may receive nested calls by | ||
1717 | gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit | ||
1718 | URLs and by recursively calling Fetch.unpack from new (nested) Fetch | ||
1719 | instances. | ||
1720 | """ | ||
1721 | return | ||
1722 | def start_url(self, url): | ||
1723 | """Start tracing url unpack process. | ||
1724 | This method is called by Fetch.unpack before the fetcher-specific unpack | ||
1725 | method starts, and it may receive nested calls by gitsm and npmsw | ||
1726 | fetchers. | ||
1727 | """ | ||
1728 | return | ||
1729 | def unpack(self, unpack_type, destdir): | ||
1730 | """ | ||
1731 | Set unpack_type and destdir for current url. | ||
1732 | This method is called by the fetcher-specific unpack method after url | ||
1733 | tracing started. | ||
1734 | """ | ||
1735 | return | ||
1736 | def finish_url(self, url): | ||
1737 | """Finish tracing url unpack process and update the file index. | ||
1738 | This method is called by Fetch.unpack after the fetcher-specific unpack | ||
1739 | method finished its job, and it may receive nested calls by gitsm | ||
1740 | and npmsw fetchers. | ||
1741 | """ | ||
1742 | return | ||
1743 | def complete(self): | ||
1744 | """ | ||
1745 | Finish tracing the Fetch.unpack process, and check if all nested | ||
1746 | Fecth.unpack calls (if any) have been completed; if so, save collected | ||
1747 | metadata. | ||
1748 | """ | ||
1749 | return | ||
1750 | |||
1751 | |||
1614 | class Fetch(object): | 1752 | class Fetch(object): |
1615 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): | 1753 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): |
1616 | if localonly and cache: | 1754 | if localonly and cache: |
1617 | raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") | 1755 | raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") |
1618 | 1756 | ||
1619 | if len(urls) == 0: | 1757 | if not urls: |
1620 | urls = d.getVar("SRC_URI").split() | 1758 | urls = d.getVar("SRC_URI").split() |
1621 | self.urls = urls | 1759 | self.urls = urls |
1622 | self.d = d | 1760 | self.d = d |
@@ -1631,10 +1769,30 @@ class Fetch(object): | |||
1631 | if key in urldata_cache: | 1769 | if key in urldata_cache: |
1632 | self.ud = urldata_cache[key] | 1770 | self.ud = urldata_cache[key] |
1633 | 1771 | ||
1772 | # the unpack_tracer object needs to be made available to possible nested | ||
1773 | # Fetch instances (when those are created by gitsm and npmsw fetchers) | ||
1774 | # so we set it as a global variable | ||
1775 | global unpack_tracer | ||
1776 | try: | ||
1777 | unpack_tracer | ||
1778 | except NameError: | ||
1779 | class_path = d.getVar("BB_UNPACK_TRACER_CLASS") | ||
1780 | if class_path: | ||
1781 | # use user-defined unpack tracer class | ||
1782 | import importlib | ||
1783 | module_name, _, class_name = class_path.rpartition(".") | ||
1784 | module = importlib.import_module(module_name) | ||
1785 | class_ = getattr(module, class_name) | ||
1786 | unpack_tracer = class_() | ||
1787 | else: | ||
1788 | # fall back to the dummy/abstract class | ||
1789 | unpack_tracer = DummyUnpackTracer() | ||
1790 | |||
1634 | for url in urls: | 1791 | for url in urls: |
1635 | if url not in self.ud: | 1792 | if url not in self.ud: |
1636 | try: | 1793 | try: |
1637 | self.ud[url] = FetchData(url, d, localonly) | 1794 | self.ud[url] = FetchData(url, d, localonly) |
1795 | self.ud[url].unpack_tracer = unpack_tracer | ||
1638 | except NonLocalMethod: | 1796 | except NonLocalMethod: |
1639 | if localonly: | 1797 | if localonly: |
1640 | self.ud[url] = None | 1798 | self.ud[url] = None |
@@ -1673,6 +1831,7 @@ class Fetch(object): | |||
1673 | network = self.d.getVar("BB_NO_NETWORK") | 1831 | network = self.d.getVar("BB_NO_NETWORK") |
1674 | premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) | 1832 | premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) |
1675 | 1833 | ||
1834 | checksum_missing_messages = [] | ||
1676 | for u in urls: | 1835 | for u in urls: |
1677 | ud = self.ud[u] | 1836 | ud = self.ud[u] |
1678 | ud.setup_localpath(self.d) | 1837 | ud.setup_localpath(self.d) |
@@ -1684,7 +1843,6 @@ class Fetch(object): | |||
1684 | 1843 | ||
1685 | try: | 1844 | try: |
1686 | self.d.setVar("BB_NO_NETWORK", network) | 1845 | self.d.setVar("BB_NO_NETWORK", network) |
1687 | |||
1688 | if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): | 1846 | if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): |
1689 | done = True | 1847 | done = True |
1690 | elif m.try_premirror(ud, self.d): | 1848 | elif m.try_premirror(ud, self.d): |
@@ -1705,7 +1863,9 @@ class Fetch(object): | |||
1705 | self.d.setVar("BB_NO_NETWORK", "1") | 1863 | self.d.setVar("BB_NO_NETWORK", "1") |
1706 | 1864 | ||
1707 | firsterr = None | 1865 | firsterr = None |
1708 | verified_stamp = m.verify_donestamp(ud, self.d) | 1866 | verified_stamp = False |
1867 | if done: | ||
1868 | verified_stamp = m.verify_donestamp(ud, self.d) | ||
1709 | if not done and (not verified_stamp or m.need_update(ud, self.d)): | 1869 | if not done and (not verified_stamp or m.need_update(ud, self.d)): |
1710 | try: | 1870 | try: |
1711 | if not trusted_network(self.d, ud.url): | 1871 | if not trusted_network(self.d, ud.url): |
@@ -1735,7 +1895,7 @@ class Fetch(object): | |||
1735 | logger.debug(str(e)) | 1895 | logger.debug(str(e)) |
1736 | firsterr = e | 1896 | firsterr = e |
1737 | # Remove any incomplete fetch | 1897 | # Remove any incomplete fetch |
1738 | if not verified_stamp: | 1898 | if not verified_stamp and m.cleanup_upon_failure(): |
1739 | m.clean(ud, self.d) | 1899 | m.clean(ud, self.d) |
1740 | logger.debug("Trying MIRRORS") | 1900 | logger.debug("Trying MIRRORS") |
1741 | mirrors = mirror_from_string(self.d.getVar('MIRRORS')) | 1901 | mirrors = mirror_from_string(self.d.getVar('MIRRORS')) |
@@ -1754,17 +1914,28 @@ class Fetch(object): | |||
1754 | raise ChecksumError("Stale Error Detected") | 1914 | raise ChecksumError("Stale Error Detected") |
1755 | 1915 | ||
1756 | except BBFetchException as e: | 1916 | except BBFetchException as e: |
1757 | if isinstance(e, ChecksumError): | 1917 | if isinstance(e, NoChecksumError): |
1918 | (message, _) = e.args | ||
1919 | checksum_missing_messages.append(message) | ||
1920 | continue | ||
1921 | elif isinstance(e, ChecksumError): | ||
1758 | logger.error("Checksum failure fetching %s" % u) | 1922 | logger.error("Checksum failure fetching %s" % u) |
1759 | raise | 1923 | raise |
1760 | 1924 | ||
1761 | finally: | 1925 | finally: |
1762 | if ud.lockfile: | 1926 | if ud.lockfile: |
1763 | bb.utils.unlockfile(lf) | 1927 | bb.utils.unlockfile(lf) |
1928 | if checksum_missing_messages: | ||
1929 | logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages)) | ||
1930 | raise BBFetchException("There was some missing checksums in the recipe") | ||
1764 | 1931 | ||
1765 | def checkstatus(self, urls=None): | 1932 | def checkstatus(self, urls=None): |
1766 | """ | 1933 | """ |
1767 | Check all urls exist upstream | 1934 | Check all URLs exist upstream. |
1935 | |||
1936 | Returns None if the URLs exist, raises FetchError if the check wasn't | ||
1937 | successful but there wasn't an error (such as file not found), and | ||
1938 | raises other exceptions in error cases. | ||
1768 | """ | 1939 | """ |
1769 | 1940 | ||
1770 | if not urls: | 1941 | if not urls: |
@@ -1787,7 +1958,7 @@ class Fetch(object): | |||
1787 | ret = m.try_mirrors(self, ud, self.d, mirrors, True) | 1958 | ret = m.try_mirrors(self, ud, self.d, mirrors, True) |
1788 | 1959 | ||
1789 | if not ret: | 1960 | if not ret: |
1790 | raise FetchError("URL %s doesn't work" % u, u) | 1961 | raise FetchError("URL doesn't work", u) |
1791 | 1962 | ||
1792 | def unpack(self, root, urls=None): | 1963 | def unpack(self, root, urls=None): |
1793 | """ | 1964 | """ |
@@ -1797,6 +1968,8 @@ class Fetch(object): | |||
1797 | if not urls: | 1968 | if not urls: |
1798 | urls = self.urls | 1969 | urls = self.urls |
1799 | 1970 | ||
1971 | unpack_tracer.start(root, self.ud, self.d) | ||
1972 | |||
1800 | for u in urls: | 1973 | for u in urls: |
1801 | ud = self.ud[u] | 1974 | ud = self.ud[u] |
1802 | ud.setup_localpath(self.d) | 1975 | ud.setup_localpath(self.d) |
@@ -1804,11 +1977,15 @@ class Fetch(object): | |||
1804 | if ud.lockfile: | 1977 | if ud.lockfile: |
1805 | lf = bb.utils.lockfile(ud.lockfile) | 1978 | lf = bb.utils.lockfile(ud.lockfile) |
1806 | 1979 | ||
1980 | unpack_tracer.start_url(u) | ||
1807 | ud.method.unpack(ud, root, self.d) | 1981 | ud.method.unpack(ud, root, self.d) |
1982 | unpack_tracer.finish_url(u) | ||
1808 | 1983 | ||
1809 | if ud.lockfile: | 1984 | if ud.lockfile: |
1810 | bb.utils.unlockfile(lf) | 1985 | bb.utils.unlockfile(lf) |
1811 | 1986 | ||
1987 | unpack_tracer.complete() | ||
1988 | |||
1812 | def clean(self, urls=None): | 1989 | def clean(self, urls=None): |
1813 | """ | 1990 | """ |
1814 | Clean files that the fetcher gets or places | 1991 | Clean files that the fetcher gets or places |
@@ -1908,6 +2085,9 @@ from . import repo | |||
1908 | from . import clearcase | 2085 | from . import clearcase |
1909 | from . import npm | 2086 | from . import npm |
1910 | from . import npmsw | 2087 | from . import npmsw |
2088 | from . import az | ||
2089 | from . import crate | ||
2090 | from . import gcp | ||
1911 | 2091 | ||
1912 | methods.append(local.Local()) | 2092 | methods.append(local.Local()) |
1913 | methods.append(wget.Wget()) | 2093 | methods.append(wget.Wget()) |
@@ -1927,3 +2107,6 @@ methods.append(repo.Repo()) | |||
1927 | methods.append(clearcase.ClearCase()) | 2107 | methods.append(clearcase.ClearCase()) |
1928 | methods.append(npm.Npm()) | 2108 | methods.append(npm.Npm()) |
1929 | methods.append(npmsw.NpmShrinkWrap()) | 2109 | methods.append(npmsw.NpmShrinkWrap()) |
2110 | methods.append(az.Az()) | ||
2111 | methods.append(crate.Crate()) | ||
2112 | methods.append(gcp.GCP()) | ||
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py new file mode 100644 index 0000000000..3ccc594c22 --- /dev/null +++ b/bitbake/lib/bb/fetch2/az.py | |||
@@ -0,0 +1,93 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' Azure Storage implementation | ||
3 | |||
4 | """ | ||
5 | |||
6 | # Copyright (C) 2021 Alejandro Hernandez Samaniego | ||
7 | # | ||
8 | # Based on bb.fetch2.wget: | ||
9 | # Copyright (C) 2003, 2004 Chris Larson | ||
10 | # | ||
11 | # SPDX-License-Identifier: GPL-2.0-only | ||
12 | # | ||
13 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
14 | |||
15 | import shlex | ||
16 | import os | ||
17 | import bb | ||
18 | from bb.fetch2 import FetchError | ||
19 | from bb.fetch2 import logger | ||
20 | from bb.fetch2.wget import Wget | ||
21 | |||
22 | |||
23 | class Az(Wget): | ||
24 | |||
25 | def supports(self, ud, d): | ||
26 | """ | ||
27 | Check to see if a given url can be fetched from Azure Storage | ||
28 | """ | ||
29 | return ud.type in ['az'] | ||
30 | |||
31 | |||
32 | def checkstatus(self, fetch, ud, d, try_again=True): | ||
33 | |||
34 | # checkstatus discards parameters either way, we need to do this before adding the SAS | ||
35 | ud.url = ud.url.replace('az://','https://').split(';')[0] | ||
36 | |||
37 | az_sas = d.getVar('AZ_SAS') | ||
38 | if az_sas and az_sas not in ud.url: | ||
39 | ud.url += az_sas | ||
40 | |||
41 | return Wget.checkstatus(self, fetch, ud, d, try_again) | ||
42 | |||
43 | # Override download method, include retries | ||
44 | def download(self, ud, d, retries=3): | ||
45 | """Fetch urls""" | ||
46 | |||
47 | # If were reaching the account transaction limit we might be refused a connection, | ||
48 | # retrying allows us to avoid false negatives since the limit changes over time | ||
49 | fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' | ||
50 | |||
51 | # We need to provide a localpath to avoid wget using the SAS | ||
52 | # ud.localfile either has the downloadfilename or ud.path | ||
53 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | ||
54 | bb.utils.mkdirhier(os.path.dirname(localpath)) | ||
55 | fetchcmd += " -O %s" % shlex.quote(localpath) | ||
56 | |||
57 | |||
58 | if ud.user and ud.pswd: | ||
59 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | ||
60 | |||
61 | # Check if a Shared Access Signature was given and use it | ||
62 | az_sas = d.getVar('AZ_SAS') | ||
63 | |||
64 | if az_sas: | ||
65 | azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) | ||
66 | else: | ||
67 | azuri = '%s%s%s' % ('https://', ud.host, ud.path) | ||
68 | |||
69 | if os.path.exists(ud.localpath): | ||
70 | # file exists, but we didnt complete it.. trying again. | ||
71 | fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri) | ||
72 | else: | ||
73 | fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri) | ||
74 | |||
75 | try: | ||
76 | self._runwget(ud, d, fetchcmd, False) | ||
77 | except FetchError as e: | ||
78 | # Azure fails on handshake sometimes when using wget after some stress, producing a | ||
79 | # FetchError from the fetcher, if the artifact exists retyring should succeed | ||
80 | if 'Unable to establish SSL connection' in str(e): | ||
81 | logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) | ||
82 | self.download(ud, d, retries -1) | ||
83 | |||
84 | # Sanity check since wget can pretend it succeed when it didn't | ||
85 | # Also, this used to happen if sourceforge sent us to the mirror page | ||
86 | if not os.path.exists(ud.localpath): | ||
87 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) | ||
88 | |||
89 | if os.path.getsize(ud.localpath) == 0: | ||
90 | os.remove(ud.localpath) | ||
91 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) | ||
92 | |||
93 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py new file mode 100644 index 0000000000..e611736f06 --- /dev/null +++ b/bitbake/lib/bb/fetch2/crate.py | |||
@@ -0,0 +1,150 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake 'Fetch' implementation for crates.io | ||
5 | """ | ||
6 | |||
7 | # Copyright (C) 2016 Doug Goldstein | ||
8 | # | ||
9 | # SPDX-License-Identifier: GPL-2.0-only | ||
10 | # | ||
11 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
12 | |||
13 | import hashlib | ||
14 | import json | ||
15 | import os | ||
16 | import subprocess | ||
17 | import bb | ||
18 | from bb.fetch2 import logger, subprocess_setup, UnpackError | ||
19 | from bb.fetch2.wget import Wget | ||
20 | |||
21 | |||
22 | class Crate(Wget): | ||
23 | |||
24 | """Class to fetch crates via wget""" | ||
25 | |||
26 | def _cargo_bitbake_path(self, rootdir): | ||
27 | return os.path.join(rootdir, "cargo_home", "bitbake") | ||
28 | |||
29 | def supports(self, ud, d): | ||
30 | """ | ||
31 | Check to see if a given url is for this fetcher | ||
32 | """ | ||
33 | return ud.type in ['crate'] | ||
34 | |||
35 | def recommends_checksum(self, urldata): | ||
36 | return True | ||
37 | |||
38 | def urldata_init(self, ud, d): | ||
39 | """ | ||
40 | Sets up to download the respective crate from crates.io | ||
41 | """ | ||
42 | |||
43 | if ud.type == 'crate': | ||
44 | self._crate_urldata_init(ud, d) | ||
45 | |||
46 | super(Crate, self).urldata_init(ud, d) | ||
47 | |||
48 | def _crate_urldata_init(self, ud, d): | ||
49 | """ | ||
50 | Sets up the download for a crate | ||
51 | """ | ||
52 | |||
53 | # URL syntax is: crate://NAME/VERSION | ||
54 | # break the URL apart by / | ||
55 | parts = ud.url.split('/') | ||
56 | if len(parts) < 5: | ||
57 | raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url) | ||
58 | |||
59 | # version is expected to be the last token | ||
60 | # but ignore possible url parameters which will be used | ||
61 | # by the top fetcher class | ||
62 | version = parts[-1].split(";")[0] | ||
63 | # second to last field is name | ||
64 | name = parts[-2] | ||
65 | # host (this is to allow custom crate registries to be specified | ||
66 | host = '/'.join(parts[2:-2]) | ||
67 | |||
68 | # if using upstream just fix it up nicely | ||
69 | if host == 'crates.io': | ||
70 | host = 'crates.io/api/v1/crates' | ||
71 | |||
72 | ud.url = "https://%s/%s/%s/download" % (host, name, version) | ||
73 | ud.versionsurl = "https://%s/%s/versions" % (host, name) | ||
74 | ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) | ||
75 | if 'name' not in ud.parm: | ||
76 | ud.parm['name'] = '%s-%s' % (name, version) | ||
77 | |||
78 | logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename'])) | ||
79 | |||
80 | def unpack(self, ud, rootdir, d): | ||
81 | """ | ||
82 | Uses the crate to build the necessary paths for cargo to utilize it | ||
83 | """ | ||
84 | if ud.type == 'crate': | ||
85 | return self._crate_unpack(ud, rootdir, d) | ||
86 | else: | ||
87 | super(Crate, self).unpack(ud, rootdir, d) | ||
88 | |||
89 | def _crate_unpack(self, ud, rootdir, d): | ||
90 | """ | ||
91 | Unpacks a crate | ||
92 | """ | ||
93 | thefile = ud.localpath | ||
94 | |||
95 | # possible metadata we need to write out | ||
96 | metadata = {} | ||
97 | |||
98 | # change to the rootdir to unpack but save the old working dir | ||
99 | save_cwd = os.getcwd() | ||
100 | os.chdir(rootdir) | ||
101 | |||
102 | bp = d.getVar('BP') | ||
103 | if bp == ud.parm.get('name'): | ||
104 | cmd = "tar -xz --no-same-owner -f %s" % thefile | ||
105 | ud.unpack_tracer.unpack("crate-extract", rootdir) | ||
106 | else: | ||
107 | cargo_bitbake = self._cargo_bitbake_path(rootdir) | ||
108 | ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake) | ||
109 | |||
110 | cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) | ||
111 | |||
112 | # ensure we've got these paths made | ||
113 | bb.utils.mkdirhier(cargo_bitbake) | ||
114 | |||
115 | # generate metadata necessary | ||
116 | with open(thefile, 'rb') as f: | ||
117 | # get the SHA256 of the original tarball | ||
118 | tarhash = hashlib.sha256(f.read()).hexdigest() | ||
119 | |||
120 | metadata['files'] = {} | ||
121 | metadata['package'] = tarhash | ||
122 | |||
123 | path = d.getVar('PATH') | ||
124 | if path: | ||
125 | cmd = "PATH=\"%s\" %s" % (path, cmd) | ||
126 | bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) | ||
127 | |||
128 | ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) | ||
129 | |||
130 | os.chdir(save_cwd) | ||
131 | |||
132 | if ret != 0: | ||
133 | raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) | ||
134 | |||
135 | # if we have metadata to write out.. | ||
136 | if len(metadata) > 0: | ||
137 | cratepath = os.path.splitext(os.path.basename(thefile))[0] | ||
138 | bbpath = self._cargo_bitbake_path(rootdir) | ||
139 | mdfile = '.cargo-checksum.json' | ||
140 | mdpath = os.path.join(bbpath, cratepath, mdfile) | ||
141 | with open(mdpath, "w") as f: | ||
142 | json.dump(metadata, f) | ||
143 | |||
144 | def latest_versionstring(self, ud, d): | ||
145 | from functools import cmp_to_key | ||
146 | json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d)) | ||
147 | versions = [(0, i["num"], "") for i in json_data["versions"]] | ||
148 | versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) | ||
149 | |||
150 | return (versions[-1][1], "") | ||
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py new file mode 100644 index 0000000000..eb3e0c6a6b --- /dev/null +++ b/bitbake/lib/bb/fetch2/gcp.py | |||
@@ -0,0 +1,102 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' implementation for Google Cloup Platform Storage. | ||
3 | |||
4 | Class for fetching files from Google Cloud Storage using the | ||
5 | Google Cloud Storage Python Client. The GCS Python Client must | ||
6 | be correctly installed, configured and authenticated prior to use. | ||
7 | Additionally, gsutil must also be installed. | ||
8 | |||
9 | """ | ||
10 | |||
11 | # Copyright (C) 2023, Snap Inc. | ||
12 | # | ||
13 | # Based in part on bb.fetch2.s3: | ||
14 | # Copyright (C) 2017 Andre McCurdy | ||
15 | # | ||
16 | # SPDX-License-Identifier: GPL-2.0-only | ||
17 | # | ||
18 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
19 | |||
20 | import os | ||
21 | import bb | ||
22 | import urllib.parse, urllib.error | ||
23 | from bb.fetch2 import FetchMethod | ||
24 | from bb.fetch2 import FetchError | ||
25 | from bb.fetch2 import logger | ||
26 | from bb.fetch2 import runfetchcmd | ||
27 | |||
28 | class GCP(FetchMethod): | ||
29 | """ | ||
30 | Class to fetch urls via GCP's Python API. | ||
31 | """ | ||
32 | def __init__(self): | ||
33 | self.gcp_client = None | ||
34 | |||
35 | def supports(self, ud, d): | ||
36 | """ | ||
37 | Check to see if a given url can be fetched with GCP. | ||
38 | """ | ||
39 | return ud.type in ['gs'] | ||
40 | |||
41 | def recommends_checksum(self, urldata): | ||
42 | return True | ||
43 | |||
44 | def urldata_init(self, ud, d): | ||
45 | if 'downloadfilename' in ud.parm: | ||
46 | ud.basename = ud.parm['downloadfilename'] | ||
47 | else: | ||
48 | ud.basename = os.path.basename(ud.path) | ||
49 | |||
50 | ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) | ||
51 | ud.basecmd = "gsutil stat" | ||
52 | |||
53 | def get_gcp_client(self): | ||
54 | from google.cloud import storage | ||
55 | self.gcp_client = storage.Client(project=None) | ||
56 | |||
57 | def download(self, ud, d): | ||
58 | """ | ||
59 | Fetch urls using the GCP API. | ||
60 | Assumes localpath was called first. | ||
61 | """ | ||
62 | logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}") | ||
63 | if self.gcp_client is None: | ||
64 | self.get_gcp_client() | ||
65 | |||
66 | bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}") | ||
67 | runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d) | ||
68 | |||
69 | # Path sometimes has leading slash, so strip it | ||
70 | path = ud.path.lstrip("/") | ||
71 | blob = self.gcp_client.bucket(ud.host).blob(path) | ||
72 | blob.download_to_filename(ud.localpath) | ||
73 | |||
74 | # Additional sanity checks copied from the wget class (although there | ||
75 | # are no known issues which mean these are required, treat the GCP API | ||
76 | # tool with a little healthy suspicion). | ||
77 | if not os.path.exists(ud.localpath): | ||
78 | raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!") | ||
79 | |||
80 | if os.path.getsize(ud.localpath) == 0: | ||
81 | os.remove(ud.localpath) | ||
82 | raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.") | ||
83 | |||
84 | return True | ||
85 | |||
86 | def checkstatus(self, fetch, ud, d): | ||
87 | """ | ||
88 | Check the status of a URL. | ||
89 | """ | ||
90 | logger.debug2(f"Checking status of gs://{ud.host}{ud.path}") | ||
91 | if self.gcp_client is None: | ||
92 | self.get_gcp_client() | ||
93 | |||
94 | bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}") | ||
95 | runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d) | ||
96 | |||
97 | # Path sometimes has leading slash, so strip it | ||
98 | path = ud.path.lstrip("/") | ||
99 | if self.gcp_client.bucket(ud.host).blob(path).exists() == False: | ||
100 | raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist") | ||
101 | else: | ||
102 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index e3ba80a3f5..c7ff769fdf 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -44,13 +44,27 @@ Supported SRC_URI options are: | |||
44 | 44 | ||
45 | - nobranch | 45 | - nobranch |
46 | Don't check the SHA validation for branch. set this option for the recipe | 46 | Don't check the SHA validation for branch. set this option for the recipe |
47 | referring to commit which is valid in tag instead of branch. | 47 | referring to commit which is valid in any namespace (branch, tag, ...) |
48 | instead of branch. | ||
48 | The default is "0", set nobranch=1 if needed. | 49 | The default is "0", set nobranch=1 if needed. |
49 | 50 | ||
51 | - subpath | ||
52 | Limit the checkout to a specific subpath of the tree. | ||
53 | By default, checkout the whole tree, set subpath=<path> if needed | ||
54 | |||
55 | - destsuffix | ||
56 | The name of the path in which to place the checkout. | ||
57 | By default, the path is git/, set destsuffix=<suffix> if needed | ||
58 | |||
50 | - usehead | 59 | - usehead |
51 | For local git:// urls to use the current branch HEAD as the revision for use with | 60 | For local git:// urls to use the current branch HEAD as the revision for use with |
52 | AUTOREV. Implies nobranch. | 61 | AUTOREV. Implies nobranch. |
53 | 62 | ||
63 | - lfs | ||
64 | Enable the checkout to use LFS for large files. This will download all LFS files | ||
65 | in the download step, as the unpack step does not have network access. | ||
66 | The default is "1", set lfs=0 to skip. | ||
67 | |||
54 | """ | 68 | """ |
55 | 69 | ||
56 | # Copyright (C) 2005 Richard Purdie | 70 | # Copyright (C) 2005 Richard Purdie |
@@ -64,14 +78,20 @@ import fnmatch | |||
64 | import os | 78 | import os |
65 | import re | 79 | import re |
66 | import shlex | 80 | import shlex |
81 | import shutil | ||
67 | import subprocess | 82 | import subprocess |
68 | import tempfile | 83 | import tempfile |
69 | import bb | 84 | import bb |
70 | import bb.progress | 85 | import bb.progress |
86 | from contextlib import contextmanager | ||
71 | from bb.fetch2 import FetchMethod | 87 | from bb.fetch2 import FetchMethod |
72 | from bb.fetch2 import runfetchcmd | 88 | from bb.fetch2 import runfetchcmd |
73 | from bb.fetch2 import logger | 89 | from bb.fetch2 import logger |
90 | from bb.fetch2 import trusted_network | ||
91 | |||
74 | 92 | ||
93 | sha1_re = re.compile(r'^[0-9a-f]{40}$') | ||
94 | slash_re = re.compile(r"/+") | ||
75 | 95 | ||
76 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): | 96 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): |
77 | """Extract progress information from git output""" | 97 | """Extract progress information from git output""" |
@@ -130,6 +150,9 @@ class Git(FetchMethod): | |||
130 | def supports_checksum(self, urldata): | 150 | def supports_checksum(self, urldata): |
131 | return False | 151 | return False |
132 | 152 | ||
153 | def cleanup_upon_failure(self): | ||
154 | return False | ||
155 | |||
133 | def urldata_init(self, ud, d): | 156 | def urldata_init(self, ud, d): |
134 | """ | 157 | """ |
135 | init git specific variable within url data | 158 | init git specific variable within url data |
@@ -141,6 +164,11 @@ class Git(FetchMethod): | |||
141 | ud.proto = 'file' | 164 | ud.proto = 'file' |
142 | else: | 165 | else: |
143 | ud.proto = "git" | 166 | ud.proto = "git" |
167 | if ud.host == "github.com" and ud.proto == "git": | ||
168 | # github stopped supporting git protocol | ||
169 | # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git | ||
170 | ud.proto = "https" | ||
171 | bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url) | ||
144 | 172 | ||
145 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | 173 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
146 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | 174 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
@@ -164,11 +192,18 @@ class Git(FetchMethod): | |||
164 | ud.nocheckout = 1 | 192 | ud.nocheckout = 1 |
165 | 193 | ||
166 | ud.unresolvedrev = {} | 194 | ud.unresolvedrev = {} |
167 | branches = ud.parm.get("branch", "master").split(',') | 195 | branches = ud.parm.get("branch", "").split(',') |
196 | if branches == [""] and not ud.nobranch: | ||
197 | bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url) | ||
198 | branches = ["master"] | ||
168 | if len(branches) != len(ud.names): | 199 | if len(branches) != len(ud.names): |
169 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) | 200 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) |
170 | 201 | ||
171 | ud.cloneflags = "-s -n" | 202 | ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" |
203 | |||
204 | ud.cloneflags = "-n" | ||
205 | if not ud.noshared: | ||
206 | ud.cloneflags += " -s" | ||
172 | if ud.bareclone: | 207 | if ud.bareclone: |
173 | ud.cloneflags += " --mirror" | 208 | ud.cloneflags += " --mirror" |
174 | 209 | ||
@@ -227,7 +262,7 @@ class Git(FetchMethod): | |||
227 | for name in ud.names: | 262 | for name in ud.names: |
228 | ud.unresolvedrev[name] = 'HEAD' | 263 | ud.unresolvedrev[name] = 'HEAD' |
229 | 264 | ||
230 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" | 265 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all" |
231 | 266 | ||
232 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" | 267 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" |
233 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable | 268 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable |
@@ -236,20 +271,20 @@ class Git(FetchMethod): | |||
236 | ud.setup_revisions(d) | 271 | ud.setup_revisions(d) |
237 | 272 | ||
238 | for name in ud.names: | 273 | for name in ud.names: |
239 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one | 274 | # Ensure any revision that doesn't look like a SHA-1 is translated into one |
240 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): | 275 | if not sha1_re.match(ud.revisions[name] or ''): |
241 | if ud.revisions[name]: | 276 | if ud.revisions[name]: |
242 | ud.unresolvedrev[name] = ud.revisions[name] | 277 | ud.unresolvedrev[name] = ud.revisions[name] |
243 | ud.revisions[name] = self.latest_revision(ud, d, name) | 278 | ud.revisions[name] = self.latest_revision(ud, d, name) |
244 | 279 | ||
245 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) | 280 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) |
246 | if gitsrcname.startswith('.'): | 281 | if gitsrcname.startswith('.'): |
247 | gitsrcname = gitsrcname[1:] | 282 | gitsrcname = gitsrcname[1:] |
248 | 283 | ||
249 | # for rebaseable git repo, it is necessary to keep mirror tar ball | 284 | # For a rebaseable git repo, it is necessary to keep a mirror tar ball |
250 | # per revision, so that even the revision disappears from the | 285 | # per revision, so that even if the revision disappears from the |
251 | # upstream repo in the future, the mirror will remain intact and still | 286 | # upstream repo in the future, the mirror will remain intact and still |
252 | # contains the revision | 287 | # contain the revision |
253 | if ud.rebaseable: | 288 | if ud.rebaseable: |
254 | for name in ud.names: | 289 | for name in ud.names: |
255 | gitsrcname = gitsrcname + '_' + ud.revisions[name] | 290 | gitsrcname = gitsrcname + '_' + ud.revisions[name] |
@@ -293,7 +328,10 @@ class Git(FetchMethod): | |||
293 | return ud.clonedir | 328 | return ud.clonedir |
294 | 329 | ||
295 | def need_update(self, ud, d): | 330 | def need_update(self, ud, d): |
296 | return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) | 331 | return self.clonedir_need_update(ud, d) \ |
332 | or self.shallow_tarball_need_update(ud) \ | ||
333 | or self.tarball_need_update(ud) \ | ||
334 | or self.lfs_need_update(ud, d) | ||
297 | 335 | ||
298 | def clonedir_need_update(self, ud, d): | 336 | def clonedir_need_update(self, ud, d): |
299 | if not os.path.exists(ud.clonedir): | 337 | if not os.path.exists(ud.clonedir): |
@@ -305,6 +343,15 @@ class Git(FetchMethod): | |||
305 | return True | 343 | return True |
306 | return False | 344 | return False |
307 | 345 | ||
346 | def lfs_need_update(self, ud, d): | ||
347 | if self.clonedir_need_update(ud, d): | ||
348 | return True | ||
349 | |||
350 | for name in ud.names: | ||
351 | if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): | ||
352 | return True | ||
353 | return False | ||
354 | |||
308 | def clonedir_need_shallow_revs(self, ud, d): | 355 | def clonedir_need_shallow_revs(self, ud, d): |
309 | for rev in ud.shallow_revs: | 356 | for rev in ud.shallow_revs: |
310 | try: | 357 | try: |
@@ -324,6 +371,16 @@ class Git(FetchMethod): | |||
324 | # is not possible | 371 | # is not possible |
325 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): | 372 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): |
326 | return True | 373 | return True |
374 | # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0 | ||
375 | # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then | ||
376 | # we need to try premirrors first as using upstream is destined to fail. | ||
377 | if not trusted_network(d, ud.url): | ||
378 | return True | ||
379 | # the following check is to ensure incremental fetch in downloads, this is | ||
380 | # because the premirror might be old and does not contain the new rev required, | ||
381 | # and this will cause a total removal and new clone. So if we can reach to | ||
382 | # network, we prefer upstream over premirror, though the premirror might contain | ||
383 | # the new rev. | ||
327 | if os.path.exists(ud.clonedir): | 384 | if os.path.exists(ud.clonedir): |
328 | return False | 385 | return False |
329 | return True | 386 | return True |
@@ -337,17 +394,54 @@ class Git(FetchMethod): | |||
337 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | 394 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): |
338 | ud.localpath = ud.fullshallow | 395 | ud.localpath = ud.fullshallow |
339 | return | 396 | return |
340 | elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): | 397 | elif os.path.exists(ud.fullmirror) and self.need_update(ud, d): |
341 | bb.utils.mkdirhier(ud.clonedir) | 398 | if not os.path.exists(ud.clonedir): |
342 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | 399 | bb.utils.mkdirhier(ud.clonedir) |
343 | 400 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | |
401 | else: | ||
402 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
403 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) | ||
404 | output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) | ||
405 | if 'mirror' in output: | ||
406 | runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) | ||
407 | runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) | ||
408 | fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) | ||
409 | runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) | ||
344 | repourl = self._get_repo_url(ud) | 410 | repourl = self._get_repo_url(ud) |
345 | 411 | ||
412 | needs_clone = False | ||
413 | if os.path.exists(ud.clonedir): | ||
414 | # The directory may exist, but not be the top level of a bare git | ||
415 | # repository in which case it needs to be deleted and re-cloned. | ||
416 | try: | ||
417 | # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel | ||
418 | output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) | ||
419 | toplevel = output.rstrip() | ||
420 | |||
421 | if not bb.utils.path_is_descendant(toplevel, ud.clonedir): | ||
422 | logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir) | ||
423 | needs_clone = True | ||
424 | except bb.fetch2.FetchError as e: | ||
425 | logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) | ||
426 | needs_clone = True | ||
427 | except FileNotFoundError as e: | ||
428 | logger.warning("%s", e) | ||
429 | needs_clone = True | ||
430 | |||
431 | if needs_clone: | ||
432 | shutil.rmtree(ud.clonedir) | ||
433 | else: | ||
434 | needs_clone = True | ||
435 | |||
346 | # If the repo still doesn't exist, fallback to cloning it | 436 | # If the repo still doesn't exist, fallback to cloning it |
347 | if not os.path.exists(ud.clonedir): | 437 | if needs_clone: |
348 | # We do this since git will use a "-l" option automatically for local urls where possible | 438 | # We do this since git will use a "-l" option automatically for local urls where possible, |
439 | # but it doesn't work when git/objects is a symlink, only works when it is a directory. | ||
349 | if repourl.startswith("file://"): | 440 | if repourl.startswith("file://"): |
350 | repourl = repourl[7:] | 441 | repourl_path = repourl[7:] |
442 | objects = os.path.join(repourl_path, 'objects') | ||
443 | if os.path.isdir(objects) and not os.path.islink(objects): | ||
444 | repourl = repourl_path | ||
351 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) | 445 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) |
352 | if ud.proto.lower() != 'file': | 446 | if ud.proto.lower() != 'file': |
353 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) | 447 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) |
@@ -361,7 +455,11 @@ class Git(FetchMethod): | |||
361 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) | 455 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
362 | 456 | ||
363 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) | 457 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) |
364 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | 458 | |
459 | if ud.nobranch: | ||
460 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | ||
461 | else: | ||
462 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) | ||
365 | if ud.proto.lower() != 'file': | 463 | if ud.proto.lower() != 'file': |
366 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | 464 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
367 | progresshandler = GitProgressHandler(d) | 465 | progresshandler = GitProgressHandler(d) |
@@ -384,17 +482,16 @@ class Git(FetchMethod): | |||
384 | if missing_rev: | 482 | if missing_rev: |
385 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 483 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
386 | 484 | ||
387 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | 485 | if self.lfs_need_update(ud, d): |
388 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | 486 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching |
389 | # of all LFS blobs needed at the the srcrev. | 487 | # of all LFS blobs needed at the srcrev. |
390 | # | 488 | # |
391 | # It would be nice to just do this inline here by running 'git-lfs fetch' | 489 | # It would be nice to just do this inline here by running 'git-lfs fetch' |
392 | # on the bare clonedir, but that operation requires a working copy on some | 490 | # on the bare clonedir, but that operation requires a working copy on some |
393 | # releases of Git LFS. | 491 | # releases of Git LFS. |
394 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 492 | with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir: |
395 | try: | ||
396 | # Do the checkout. This implicitly involves a Git LFS fetch. | 493 | # Do the checkout. This implicitly involves a Git LFS fetch. |
397 | self.unpack(ud, tmpdir, d) | 494 | Git.unpack(self, ud, tmpdir, d) |
398 | 495 | ||
399 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | 496 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into |
400 | # the bare clonedir. | 497 | # the bare clonedir. |
@@ -408,12 +505,24 @@ class Git(FetchMethod): | |||
408 | # Only do this if the unpack resulted in a .git/lfs directory being | 505 | # Only do this if the unpack resulted in a .git/lfs directory being |
409 | # created; this only happens if at least one blob needed to be | 506 | # created; this only happens if at least one blob needed to be |
410 | # downloaded. | 507 | # downloaded. |
411 | if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): | 508 | if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")): |
412 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) | 509 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir) |
413 | finally: | ||
414 | bb.utils.remove(tmpdir, recurse=True) | ||
415 | 510 | ||
416 | def build_mirror_data(self, ud, d): | 511 | def build_mirror_data(self, ud, d): |
512 | |||
513 | # Create as a temp file and move atomically into position to avoid races | ||
514 | @contextmanager | ||
515 | def create_atomic(filename): | ||
516 | fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) | ||
517 | try: | ||
518 | yield tfile | ||
519 | umask = os.umask(0o666) | ||
520 | os.umask(umask) | ||
521 | os.chmod(tfile, (0o666 & ~umask)) | ||
522 | os.rename(tfile, filename) | ||
523 | finally: | ||
524 | os.close(fd) | ||
525 | |||
417 | if ud.shallow and ud.write_shallow_tarballs: | 526 | if ud.shallow and ud.write_shallow_tarballs: |
418 | if not os.path.exists(ud.fullshallow): | 527 | if not os.path.exists(ud.fullshallow): |
419 | if os.path.islink(ud.fullshallow): | 528 | if os.path.islink(ud.fullshallow): |
@@ -424,7 +533,8 @@ class Git(FetchMethod): | |||
424 | self.clone_shallow_local(ud, shallowclone, d) | 533 | self.clone_shallow_local(ud, shallowclone, d) |
425 | 534 | ||
426 | logger.info("Creating tarball of git repository") | 535 | logger.info("Creating tarball of git repository") |
427 | runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) | 536 | with create_atomic(ud.fullshallow) as tfile: |
537 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) | ||
428 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | 538 | runfetchcmd("touch %s.done" % ud.fullshallow, d) |
429 | finally: | 539 | finally: |
430 | bb.utils.remove(tempdir, recurse=True) | 540 | bb.utils.remove(tempdir, recurse=True) |
@@ -433,7 +543,11 @@ class Git(FetchMethod): | |||
433 | os.unlink(ud.fullmirror) | 543 | os.unlink(ud.fullmirror) |
434 | 544 | ||
435 | logger.info("Creating tarball of git repository") | 545 | logger.info("Creating tarball of git repository") |
436 | runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) | 546 | with create_atomic(ud.fullmirror) as tfile: |
547 | mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d, | ||
548 | quiet=True, workdir=ud.clonedir) | ||
549 | runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ." | ||
550 | % (tfile, mtime), d, workdir=ud.clonedir) | ||
437 | runfetchcmd("touch %s.done" % ud.fullmirror, d) | 551 | runfetchcmd("touch %s.done" % ud.fullmirror, d) |
438 | 552 | ||
439 | def clone_shallow_local(self, ud, dest, d): | 553 | def clone_shallow_local(self, ud, dest, d): |
@@ -495,18 +609,31 @@ class Git(FetchMethod): | |||
495 | def unpack(self, ud, destdir, d): | 609 | def unpack(self, ud, destdir, d): |
496 | """ unpack the downloaded src to destdir""" | 610 | """ unpack the downloaded src to destdir""" |
497 | 611 | ||
498 | subdir = ud.parm.get("subpath", "") | 612 | subdir = ud.parm.get("subdir") |
499 | if subdir != "": | 613 | subpath = ud.parm.get("subpath") |
500 | readpathspec = ":%s" % subdir | 614 | readpathspec = "" |
501 | def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) | 615 | def_destsuffix = "git/" |
502 | else: | 616 | |
503 | readpathspec = "" | 617 | if subpath: |
504 | def_destsuffix = "git/" | 618 | readpathspec = ":%s" % subpath |
619 | def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/')) | ||
620 | |||
621 | if subdir: | ||
622 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | ||
623 | if os.path.isabs(subdir): | ||
624 | if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)): | ||
625 | raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url) | ||
626 | destdir = subdir | ||
627 | else: | ||
628 | destdir = os.path.join(destdir, subdir) | ||
629 | def_destsuffix = "" | ||
505 | 630 | ||
506 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) | 631 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) |
507 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | 632 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
508 | if os.path.exists(destdir): | 633 | if os.path.exists(destdir): |
509 | bb.utils.prunedir(destdir) | 634 | bb.utils.prunedir(destdir) |
635 | if not ud.bareclone: | ||
636 | ud.unpack_tracer.unpack("git", destdir) | ||
510 | 637 | ||
511 | need_lfs = self._need_lfs(ud) | 638 | need_lfs = self._need_lfs(ud) |
512 | 639 | ||
@@ -516,13 +643,12 @@ class Git(FetchMethod): | |||
516 | source_found = False | 643 | source_found = False |
517 | source_error = [] | 644 | source_error = [] |
518 | 645 | ||
519 | if not source_found: | 646 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) |
520 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) | 647 | if clonedir_is_up_to_date: |
521 | if clonedir_is_up_to_date: | 648 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) |
522 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) | 649 | source_found = True |
523 | source_found = True | 650 | else: |
524 | else: | 651 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) |
525 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) | ||
526 | 652 | ||
527 | if not source_found: | 653 | if not source_found: |
528 | if ud.shallow: | 654 | if ud.shallow: |
@@ -546,9 +672,11 @@ class Git(FetchMethod): | |||
546 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) | 672 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) |
547 | elif not need_lfs: | 673 | elif not need_lfs: |
548 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) | 674 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) |
675 | else: | ||
676 | runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) | ||
549 | 677 | ||
550 | if not ud.nocheckout: | 678 | if not ud.nocheckout: |
551 | if subdir != "": | 679 | if subpath: |
552 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, | 680 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, |
553 | workdir=destdir) | 681 | workdir=destdir) |
554 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) | 682 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) |
@@ -597,6 +725,35 @@ class Git(FetchMethod): | |||
597 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 725 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
598 | return output.split()[0] != "0" | 726 | return output.split()[0] != "0" |
599 | 727 | ||
728 | def _lfs_objects_downloaded(self, ud, d, name, wd): | ||
729 | """ | ||
730 | Verifies whether the LFS objects for requested revisions have already been downloaded | ||
731 | """ | ||
732 | # Bail out early if this repository doesn't use LFS | ||
733 | if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): | ||
734 | return True | ||
735 | |||
736 | # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file | ||
737 | # existence. | ||
738 | # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git | ||
739 | cmd = "%s lfs ls-files -l %s" \ | ||
740 | % (ud.basecmd, ud.revisions[name]) | ||
741 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() | ||
742 | # Do not do any further matching if no objects are managed by LFS | ||
743 | if not output: | ||
744 | return True | ||
745 | |||
746 | # Match all lines beginning with the hexadecimal OID | ||
747 | oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") | ||
748 | for line in output.split("\n"): | ||
749 | oid = re.search(oid_regex, line) | ||
750 | if not oid: | ||
751 | bb.warn("git lfs ls-files output '%s' did not match expected format." % line) | ||
752 | if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): | ||
753 | return False | ||
754 | |||
755 | return True | ||
756 | |||
600 | def _need_lfs(self, ud): | 757 | def _need_lfs(self, ud): |
601 | return ud.parm.get("lfs", "1") == "1" | 758 | return ud.parm.get("lfs", "1") == "1" |
602 | 759 | ||
@@ -605,13 +762,11 @@ class Git(FetchMethod): | |||
605 | Check if the repository has 'lfs' (large file) content | 762 | Check if the repository has 'lfs' (large file) content |
606 | """ | 763 | """ |
607 | 764 | ||
608 | if not ud.nobranch: | 765 | if ud.nobranch: |
609 | branchname = ud.branches[ud.names[0]] | 766 | # If no branch is specified, use the current git commit |
610 | else: | 767 | refname = self._build_revision(ud, d, ud.names[0]) |
611 | branchname = "master" | 768 | elif wd == ud.clonedir: |
612 | 769 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | |
613 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | ||
614 | if wd == ud.clonedir: | ||
615 | refname = ud.branches[ud.names[0]] | 770 | refname = ud.branches[ud.names[0]] |
616 | else: | 771 | else: |
617 | refname = "origin/%s" % ud.branches[ud.names[0]] | 772 | refname = "origin/%s" % ud.branches[ud.names[0]] |
@@ -654,7 +809,6 @@ class Git(FetchMethod): | |||
654 | Return a unique key for the url | 809 | Return a unique key for the url |
655 | """ | 810 | """ |
656 | # Collapse adjacent slashes | 811 | # Collapse adjacent slashes |
657 | slash_re = re.compile(r"/+") | ||
658 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] | 812 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] |
659 | 813 | ||
660 | def _lsremote(self, ud, d, search): | 814 | def _lsremote(self, ud, d, search): |
@@ -687,6 +841,12 @@ class Git(FetchMethod): | |||
687 | """ | 841 | """ |
688 | Compute the HEAD revision for the url | 842 | Compute the HEAD revision for the url |
689 | """ | 843 | """ |
844 | if not d.getVar("__BBSRCREV_SEEN"): | ||
845 | raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path)) | ||
846 | |||
847 | # Ensure we mark as not cached | ||
848 | bb.fetch2.mark_recipe_nocache(d) | ||
849 | |||
690 | output = self._lsremote(ud, d, "") | 850 | output = self._lsremote(ud, d, "") |
691 | # Tags of the form ^{} may not work, need to fallback to other form | 851 | # Tags of the form ^{} may not work, need to fallback to other form |
692 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: | 852 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: |
@@ -711,38 +871,42 @@ class Git(FetchMethod): | |||
711 | """ | 871 | """ |
712 | pupver = ('', '') | 872 | pupver = ('', '') |
713 | 873 | ||
714 | tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
715 | try: | 874 | try: |
716 | output = self._lsremote(ud, d, "refs/tags/*") | 875 | output = self._lsremote(ud, d, "refs/tags/*") |
717 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: | 876 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: |
718 | bb.note("Could not list remote: %s" % str(e)) | 877 | bb.note("Could not list remote: %s" % str(e)) |
719 | return pupver | 878 | return pupver |
720 | 879 | ||
880 | rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)") | ||
881 | pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
882 | nonrel_re = re.compile(r"(alpha|beta|rc|final)+") | ||
883 | |||
721 | verstring = "" | 884 | verstring = "" |
722 | revision = "" | ||
723 | for line in output.split("\n"): | 885 | for line in output.split("\n"): |
724 | if not line: | 886 | if not line: |
725 | break | 887 | break |
726 | 888 | ||
727 | tag_head = line.split("/")[-1] | 889 | m = rev_tag_re.match(line) |
890 | if not m: | ||
891 | continue | ||
892 | |||
893 | (revision, tag) = m.groups() | ||
894 | |||
728 | # Ignore non-released branches | 895 | # Ignore non-released branches |
729 | m = re.search(r"(alpha|beta|rc|final)+", tag_head) | 896 | if nonrel_re.search(tag): |
730 | if m: | ||
731 | continue | 897 | continue |
732 | 898 | ||
733 | # search for version in the line | 899 | # search for version in the line |
734 | tag = tagregex.search(tag_head) | 900 | m = pver_re.search(tag) |
735 | if tag is None: | 901 | if not m: |
736 | continue | 902 | continue |
737 | 903 | ||
738 | tag = tag.group('pver') | 904 | pver = m.group('pver').replace("_", ".") |
739 | tag = tag.replace("_", ".") | ||
740 | 905 | ||
741 | if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: | 906 | if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0: |
742 | continue | 907 | continue |
743 | 908 | ||
744 | verstring = tag | 909 | verstring = pver |
745 | revision = line.split()[0] | ||
746 | pupver = (verstring, revision) | 910 | pupver = (verstring, revision) |
747 | 911 | ||
748 | return pupver | 912 | return pupver |
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index a4527bf364..f7f3af7212 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py | |||
@@ -88,9 +88,9 @@ class GitSM(Git): | |||
88 | subrevision[m] = module_hash.split()[2] | 88 | subrevision[m] = module_hash.split()[2] |
89 | 89 | ||
90 | # Convert relative to absolute uri based on parent uri | 90 | # Convert relative to absolute uri based on parent uri |
91 | if uris[m].startswith('..'): | 91 | if uris[m].startswith('..') or uris[m].startswith('./'): |
92 | newud = copy.copy(ud) | 92 | newud = copy.copy(ud) |
93 | newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) | 93 | newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) |
94 | uris[m] = Git._get_repo_url(self, newud) | 94 | uris[m] = Git._get_repo_url(self, newud) |
95 | 95 | ||
96 | for module in submodules: | 96 | for module in submodules: |
@@ -115,10 +115,21 @@ class GitSM(Git): | |||
115 | # This has to be a file reference | 115 | # This has to be a file reference |
116 | proto = "file" | 116 | proto = "file" |
117 | url = "gitsm://" + uris[module] | 117 | url = "gitsm://" + uris[module] |
118 | if url.endswith("{}{}".format(ud.host, ud.path)): | ||
119 | raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ | ||
120 | "Consider using git fetcher instead.") | ||
118 | 121 | ||
119 | url += ';protocol=%s' % proto | 122 | url += ';protocol=%s' % proto |
120 | url += ";name=%s" % module | 123 | url += ";name=%s" % module |
121 | url += ";subpath=%s" % module | 124 | url += ";subpath=%s" % module |
125 | url += ";nobranch=1" | ||
126 | url += ";lfs=%s" % self._need_lfs(ud) | ||
127 | # Note that adding "user=" here to give credentials to the | ||
128 | # submodule is not supported. Since using SRC_URI to give git:// | ||
129 | # URL a password is not supported, one have to use one of the | ||
130 | # recommended way (eg. ~/.netrc or SSH config) which does specify | ||
131 | # the user (See comment in git.py). | ||
132 | # So, we will not take patches adding "user=" support here. | ||
122 | 133 | ||
123 | ld = d.createCopy() | 134 | ld = d.createCopy() |
124 | # Not necessary to set SRC_URI, since we're passing the URI to | 135 | # Not necessary to set SRC_URI, since we're passing the URI to |
@@ -140,16 +151,6 @@ class GitSM(Git): | |||
140 | if Git.need_update(self, ud, d): | 151 | if Git.need_update(self, ud, d): |
141 | return True | 152 | return True |
142 | 153 | ||
143 | try: | ||
144 | # Check for the nugget dropped by the download operation | ||
145 | known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ | ||
146 | (ud.basecmd), d, workdir=ud.clonedir) | ||
147 | |||
148 | if ud.revisions[ud.names[0]] in known_srcrevs.split(): | ||
149 | return False | ||
150 | except bb.fetch2.FetchError: | ||
151 | pass | ||
152 | |||
153 | need_update_list = [] | 154 | need_update_list = [] |
154 | def need_update_submodule(ud, url, module, modpath, workdir, d): | 155 | def need_update_submodule(ud, url, module, modpath, workdir, d): |
155 | url += ";bareclone=1;nobranch=1" | 156 | url += ";bareclone=1;nobranch=1" |
@@ -172,13 +173,8 @@ class GitSM(Git): | |||
172 | shutil.rmtree(tmpdir) | 173 | shutil.rmtree(tmpdir) |
173 | else: | 174 | else: |
174 | self.process_submodules(ud, ud.clonedir, need_update_submodule, d) | 175 | self.process_submodules(ud, ud.clonedir, need_update_submodule, d) |
175 | if len(need_update_list) == 0: | ||
176 | # We already have the required commits of all submodules. Drop | ||
177 | # a nugget so we don't need to check again. | ||
178 | runfetchcmd("%s config --add bitbake.srcrev %s" % \ | ||
179 | (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) | ||
180 | 176 | ||
181 | if len(need_update_list) > 0: | 177 | if need_update_list: |
182 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) | 178 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) |
183 | return True | 179 | return True |
184 | 180 | ||
@@ -209,9 +205,6 @@ class GitSM(Git): | |||
209 | shutil.rmtree(tmpdir) | 205 | shutil.rmtree(tmpdir) |
210 | else: | 206 | else: |
211 | self.process_submodules(ud, ud.clonedir, download_submodule, d) | 207 | self.process_submodules(ud, ud.clonedir, download_submodule, d) |
212 | # Drop a nugget for the srcrev we've fetched (used by need_update) | ||
213 | runfetchcmd("%s config --add bitbake.srcrev %s" % \ | ||
214 | (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) | ||
215 | 208 | ||
216 | def unpack(self, ud, destdir, d): | 209 | def unpack(self, ud, destdir, d): |
217 | def unpack_submodules(ud, url, module, modpath, workdir, d): | 210 | def unpack_submodules(ud, url, module, modpath, workdir, d): |
@@ -225,6 +218,10 @@ class GitSM(Git): | |||
225 | 218 | ||
226 | try: | 219 | try: |
227 | newfetch = Fetch([url], d, cache=False) | 220 | newfetch = Fetch([url], d, cache=False) |
221 | # modpath is needed by unpack tracer to calculate submodule | ||
222 | # checkout dir | ||
223 | new_ud = newfetch.ud[url] | ||
224 | new_ud.modpath = modpath | ||
228 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) | 225 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) |
229 | except Exception as e: | 226 | except Exception as e: |
230 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) | 227 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) |
@@ -250,10 +247,12 @@ class GitSM(Git): | |||
250 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) | 247 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) |
251 | 248 | ||
252 | if not ud.bareclone and ret: | 249 | if not ud.bareclone and ret: |
253 | # All submodules should already be downloaded and configured in the tree. This simply sets | 250 | # All submodules should already be downloaded and configured in the tree. This simply |
254 | # up the configuration and checks out the files. The main project config should remain | 251 | # sets up the configuration and checks out the files. The main project config should |
255 | # unmodified, and no download from the internet should occur. | 252 | # remain unmodified, and no download from the internet should occur. As such, lfs smudge |
256 | runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) | 253 | # should also be skipped as these files were already smudged in the fetch stage if lfs |
254 | # was enabled. | ||
255 | runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) | ||
257 | 256 | ||
258 | def implicit_urldata(self, ud, d): | 257 | def implicit_urldata(self, ud, d): |
259 | import shutil, subprocess, tempfile | 258 | import shutil, subprocess, tempfile |
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index 063e13008a..cbff8c490c 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py | |||
@@ -242,6 +242,7 @@ class Hg(FetchMethod): | |||
242 | revflag = "-r %s" % ud.revision | 242 | revflag = "-r %s" % ud.revision |
243 | subdir = ud.parm.get("destsuffix", ud.module) | 243 | subdir = ud.parm.get("destsuffix", ud.module) |
244 | codir = "%s/%s" % (destdir, subdir) | 244 | codir = "%s/%s" % (destdir, subdir) |
245 | ud.unpack_tracer.unpack("hg", codir) | ||
245 | 246 | ||
246 | scmdata = ud.parm.get("scmdata", "") | 247 | scmdata = ud.parm.get("scmdata", "") |
247 | if scmdata != "nokeep": | 248 | if scmdata != "nokeep": |
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py index e7d1c8c58f..7d7668110e 100644 --- a/bitbake/lib/bb/fetch2/local.py +++ b/bitbake/lib/bb/fetch2/local.py | |||
@@ -41,9 +41,9 @@ class Local(FetchMethod): | |||
41 | """ | 41 | """ |
42 | Return the local filename of a given url assuming a successful fetch. | 42 | Return the local filename of a given url assuming a successful fetch. |
43 | """ | 43 | """ |
44 | return self.localpaths(urldata, d)[-1] | 44 | return self.localfile_searchpaths(urldata, d)[-1] |
45 | 45 | ||
46 | def localpaths(self, urldata, d): | 46 | def localfile_searchpaths(self, urldata, d): |
47 | """ | 47 | """ |
48 | Return the local filename of a given url assuming a successful fetch. | 48 | Return the local filename of a given url assuming a successful fetch. |
49 | """ | 49 | """ |
@@ -51,18 +51,14 @@ class Local(FetchMethod): | |||
51 | path = urldata.decodedurl | 51 | path = urldata.decodedurl |
52 | newpath = path | 52 | newpath = path |
53 | if path[0] == "/": | 53 | if path[0] == "/": |
54 | logger.debug2("Using absolute %s" % (path)) | ||
54 | return [path] | 55 | return [path] |
55 | filespath = d.getVar('FILESPATH') | 56 | filespath = d.getVar('FILESPATH') |
56 | if filespath: | 57 | if filespath: |
57 | logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) | 58 | logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) |
58 | newpath, hist = bb.utils.which(filespath, path, history=True) | 59 | newpath, hist = bb.utils.which(filespath, path, history=True) |
60 | logger.debug2("Using %s for %s" % (newpath, path)) | ||
59 | searched.extend(hist) | 61 | searched.extend(hist) |
60 | if not os.path.exists(newpath): | ||
61 | dldirfile = os.path.join(d.getVar("DL_DIR"), path) | ||
62 | logger.debug2("Defaulting to %s for %s" % (dldirfile, path)) | ||
63 | bb.utils.mkdirhier(os.path.dirname(dldirfile)) | ||
64 | searched.append(dldirfile) | ||
65 | return searched | ||
66 | return searched | 62 | return searched |
67 | 63 | ||
68 | def need_update(self, ud, d): | 64 | def need_update(self, ud, d): |
@@ -78,9 +74,7 @@ class Local(FetchMethod): | |||
78 | filespath = d.getVar('FILESPATH') | 74 | filespath = d.getVar('FILESPATH') |
79 | if filespath: | 75 | if filespath: |
80 | locations = filespath.split(":") | 76 | locations = filespath.split(":") |
81 | locations.append(d.getVar("DL_DIR")) | 77 | msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n " + "\n ".join(locations) |
82 | |||
83 | msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations) | ||
84 | raise FetchError(msg) | 78 | raise FetchError(msg) |
85 | 79 | ||
86 | return True | 80 | return True |
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index 47898509ff..15f3f19bc8 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py | |||
@@ -44,17 +44,24 @@ def npm_package(package): | |||
44 | """Convert the npm package name to remove unsupported character""" | 44 | """Convert the npm package name to remove unsupported character""" |
45 | # Scoped package names (with the @) use the same naming convention | 45 | # Scoped package names (with the @) use the same naming convention |
46 | # as the 'npm pack' command. | 46 | # as the 'npm pack' command. |
47 | if package.startswith("@"): | 47 | name = re.sub("/", "-", package) |
48 | return re.sub("/", "-", package[1:]) | 48 | name = name.lower() |
49 | return package | 49 | name = re.sub(r"[^\-a-z0-9]", "", name) |
50 | name = name.strip("-") | ||
51 | return name | ||
52 | |||
50 | 53 | ||
51 | def npm_filename(package, version): | 54 | def npm_filename(package, version): |
52 | """Get the filename of a npm package""" | 55 | """Get the filename of a npm package""" |
53 | return npm_package(package) + "-" + version + ".tgz" | 56 | return npm_package(package) + "-" + version + ".tgz" |
54 | 57 | ||
55 | def npm_localfile(package, version): | 58 | def npm_localfile(package, version=None): |
56 | """Get the local filename of a npm package""" | 59 | """Get the local filename of a npm package""" |
57 | return os.path.join("npm2", npm_filename(package, version)) | 60 | if version is not None: |
61 | filename = npm_filename(package, version) | ||
62 | else: | ||
63 | filename = package | ||
64 | return os.path.join("npm2", filename) | ||
58 | 65 | ||
59 | def npm_integrity(integrity): | 66 | def npm_integrity(integrity): |
60 | """ | 67 | """ |
@@ -69,41 +76,52 @@ def npm_unpack(tarball, destdir, d): | |||
69 | bb.utils.mkdirhier(destdir) | 76 | bb.utils.mkdirhier(destdir) |
70 | cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) | 77 | cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) |
71 | cmd += " --no-same-owner" | 78 | cmd += " --no-same-owner" |
79 | cmd += " --delay-directory-restore" | ||
72 | cmd += " --strip-components=1" | 80 | cmd += " --strip-components=1" |
73 | runfetchcmd(cmd, d, workdir=destdir) | 81 | runfetchcmd(cmd, d, workdir=destdir) |
82 | runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) | ||
74 | 83 | ||
75 | class NpmEnvironment(object): | 84 | class NpmEnvironment(object): |
76 | """ | 85 | """ |
77 | Using a npm config file seems more reliable than using cli arguments. | 86 | Using a npm config file seems more reliable than using cli arguments. |
78 | This class allows to create a controlled environment for npm commands. | 87 | This class allows to create a controlled environment for npm commands. |
79 | """ | 88 | """ |
80 | def __init__(self, d, configs=None): | 89 | def __init__(self, d, configs=[], npmrc=None): |
81 | self.d = d | 90 | self.d = d |
82 | self.configs = configs | 91 | |
92 | self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) | ||
93 | for key, value in configs: | ||
94 | self.user_config.write("%s=%s\n" % (key, value)) | ||
95 | |||
96 | if npmrc: | ||
97 | self.global_config_name = npmrc | ||
98 | else: | ||
99 | self.global_config_name = "/dev/null" | ||
100 | |||
101 | def __del__(self): | ||
102 | if self.user_config: | ||
103 | self.user_config.close() | ||
83 | 104 | ||
84 | def run(self, cmd, args=None, configs=None, workdir=None): | 105 | def run(self, cmd, args=None, configs=None, workdir=None): |
85 | """Run npm command in a controlled environment""" | 106 | """Run npm command in a controlled environment""" |
86 | with tempfile.TemporaryDirectory() as tmpdir: | 107 | with tempfile.TemporaryDirectory() as tmpdir: |
87 | d = bb.data.createCopy(self.d) | 108 | d = bb.data.createCopy(self.d) |
109 | d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching | ||
88 | d.setVar("HOME", tmpdir) | 110 | d.setVar("HOME", tmpdir) |
89 | 111 | ||
90 | cfgfile = os.path.join(tmpdir, "npmrc") | ||
91 | |||
92 | if not workdir: | 112 | if not workdir: |
93 | workdir = tmpdir | 113 | workdir = tmpdir |
94 | 114 | ||
95 | def _run(cmd): | 115 | def _run(cmd): |
96 | cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd | 116 | cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd |
97 | cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd | 117 | cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd |
98 | return runfetchcmd(cmd, d, workdir=workdir) | 118 | return runfetchcmd(cmd, d, workdir=workdir) |
99 | 119 | ||
100 | if self.configs: | ||
101 | for key, value in self.configs: | ||
102 | _run("npm config set %s %s" % (key, shlex.quote(value))) | ||
103 | |||
104 | if configs: | 120 | if configs: |
121 | bb.warn("Use of configs argument of NpmEnvironment.run() function" | ||
122 | " is deprecated. Please use args argument instead.") | ||
105 | for key, value in configs: | 123 | for key, value in configs: |
106 | _run("npm config set %s %s" % (key, shlex.quote(value))) | 124 | cmd += " --%s=%s" % (key, shlex.quote(value)) |
107 | 125 | ||
108 | if args: | 126 | if args: |
109 | for key, value in args: | 127 | for key, value in args: |
@@ -142,12 +160,12 @@ class Npm(FetchMethod): | |||
142 | raise ParameterError("Invalid 'version' parameter", ud.url) | 160 | raise ParameterError("Invalid 'version' parameter", ud.url) |
143 | 161 | ||
144 | # Extract the 'registry' part of the url | 162 | # Extract the 'registry' part of the url |
145 | ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) | 163 | ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) |
146 | 164 | ||
147 | # Using the 'downloadfilename' parameter as local filename | 165 | # Using the 'downloadfilename' parameter as local filename |
148 | # or the npm package name. | 166 | # or the npm package name. |
149 | if "downloadfilename" in ud.parm: | 167 | if "downloadfilename" in ud.parm: |
150 | ud.localfile = d.expand(ud.parm["downloadfilename"]) | 168 | ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) |
151 | else: | 169 | else: |
152 | ud.localfile = npm_localfile(ud.package, ud.version) | 170 | ud.localfile = npm_localfile(ud.package, ud.version) |
153 | 171 | ||
@@ -165,14 +183,14 @@ class Npm(FetchMethod): | |||
165 | 183 | ||
166 | def _resolve_proxy_url(self, ud, d): | 184 | def _resolve_proxy_url(self, ud, d): |
167 | def _npm_view(): | 185 | def _npm_view(): |
168 | configs = [] | 186 | args = [] |
169 | configs.append(("json", "true")) | 187 | args.append(("json", "true")) |
170 | configs.append(("registry", ud.registry)) | 188 | args.append(("registry", ud.registry)) |
171 | pkgver = shlex.quote(ud.package + "@" + ud.version) | 189 | pkgver = shlex.quote(ud.package + "@" + ud.version) |
172 | cmd = ud.basecmd + " view %s" % pkgver | 190 | cmd = ud.basecmd + " view %s" % pkgver |
173 | env = NpmEnvironment(d) | 191 | env = NpmEnvironment(d) |
174 | check_network_access(d, cmd, ud.registry) | 192 | check_network_access(d, cmd, ud.registry) |
175 | view_string = env.run(cmd, configs=configs) | 193 | view_string = env.run(cmd, args=args) |
176 | 194 | ||
177 | if not view_string: | 195 | if not view_string: |
178 | raise FetchError("Unavailable package %s" % pkgver, ud.url) | 196 | raise FetchError("Unavailable package %s" % pkgver, ud.url) |
@@ -280,6 +298,7 @@ class Npm(FetchMethod): | |||
280 | destsuffix = ud.parm.get("destsuffix", "npm") | 298 | destsuffix = ud.parm.get("destsuffix", "npm") |
281 | destdir = os.path.join(rootdir, destsuffix) | 299 | destdir = os.path.join(rootdir, destsuffix) |
282 | npm_unpack(ud.localpath, destdir, d) | 300 | npm_unpack(ud.localpath, destdir, d) |
301 | ud.unpack_tracer.unpack("npm", destdir) | ||
283 | 302 | ||
284 | def clean(self, ud, d): | 303 | def clean(self, ud, d): |
285 | """Clean any existing full or partial download""" | 304 | """Clean any existing full or partial download""" |
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py index 0c3511d8ab..b55e885d7b 100644 --- a/bitbake/lib/bb/fetch2/npmsw.py +++ b/bitbake/lib/bb/fetch2/npmsw.py | |||
@@ -24,11 +24,14 @@ import bb | |||
24 | from bb.fetch2 import Fetch | 24 | from bb.fetch2 import Fetch |
25 | from bb.fetch2 import FetchMethod | 25 | from bb.fetch2 import FetchMethod |
26 | from bb.fetch2 import ParameterError | 26 | from bb.fetch2 import ParameterError |
27 | from bb.fetch2 import runfetchcmd | ||
27 | from bb.fetch2 import URI | 28 | from bb.fetch2 import URI |
28 | from bb.fetch2.npm import npm_integrity | 29 | from bb.fetch2.npm import npm_integrity |
29 | from bb.fetch2.npm import npm_localfile | 30 | from bb.fetch2.npm import npm_localfile |
30 | from bb.fetch2.npm import npm_unpack | 31 | from bb.fetch2.npm import npm_unpack |
31 | from bb.utils import is_semver | 32 | from bb.utils import is_semver |
33 | from bb.utils import lockfile | ||
34 | from bb.utils import unlockfile | ||
32 | 35 | ||
33 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): | 36 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): |
34 | """ | 37 | """ |
@@ -38,8 +41,9 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False): | |||
38 | with: | 41 | with: |
39 | name = the package name (string) | 42 | name = the package name (string) |
40 | params = the package parameters (dictionary) | 43 | params = the package parameters (dictionary) |
41 | deptree = the package dependency tree (array of strings) | 44 | destdir = the destination of the package (string) |
42 | """ | 45 | """ |
46 | # For handling old style dependencies entries in shinkwrap files | ||
43 | def _walk_deps(deps, deptree): | 47 | def _walk_deps(deps, deptree): |
44 | for name in deps: | 48 | for name in deps: |
45 | subtree = [*deptree, name] | 49 | subtree = [*deptree, name] |
@@ -49,9 +53,22 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False): | |||
49 | continue | 53 | continue |
50 | elif deps[name].get("bundled", False): | 54 | elif deps[name].get("bundled", False): |
51 | continue | 55 | continue |
52 | callback(name, deps[name], subtree) | 56 | destsubdirs = [os.path.join("node_modules", dep) for dep in subtree] |
53 | 57 | destsuffix = os.path.join(*destsubdirs) | |
54 | _walk_deps(shrinkwrap.get("dependencies", {}), []) | 58 | callback(name, deps[name], destsuffix) |
59 | |||
60 | # packages entry means new style shrinkwrap file, else use dependencies | ||
61 | packages = shrinkwrap.get("packages", None) | ||
62 | if packages is not None: | ||
63 | for package in packages: | ||
64 | if package != "": | ||
65 | name = package.split('node_modules/')[-1] | ||
66 | package_infos = packages.get(package, {}) | ||
67 | if dev == False and package_infos.get("dev", False): | ||
68 | continue | ||
69 | callback(name, package_infos, package) | ||
70 | else: | ||
71 | _walk_deps(shrinkwrap.get("dependencies", {}), []) | ||
55 | 72 | ||
56 | class NpmShrinkWrap(FetchMethod): | 73 | class NpmShrinkWrap(FetchMethod): |
57 | """Class to fetch all package from a shrinkwrap file""" | 74 | """Class to fetch all package from a shrinkwrap file""" |
@@ -72,19 +89,22 @@ class NpmShrinkWrap(FetchMethod): | |||
72 | # Resolve the dependencies | 89 | # Resolve the dependencies |
73 | ud.deps = [] | 90 | ud.deps = [] |
74 | 91 | ||
75 | def _resolve_dependency(name, params, deptree): | 92 | def _resolve_dependency(name, params, destsuffix): |
76 | url = None | 93 | url = None |
77 | localpath = None | 94 | localpath = None |
78 | extrapaths = [] | 95 | extrapaths = [] |
79 | destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] | 96 | unpack = True |
80 | destsuffix = os.path.join(*destsubdirs) | ||
81 | 97 | ||
82 | integrity = params.get("integrity", None) | 98 | integrity = params.get("integrity", None) |
83 | resolved = params.get("resolved", None) | 99 | resolved = params.get("resolved", None) |
84 | version = params.get("version", None) | 100 | version = params.get("version", None) |
85 | 101 | ||
86 | # Handle registry sources | 102 | # Handle registry sources |
87 | if is_semver(version) and resolved and integrity: | 103 | if is_semver(version) and integrity: |
104 | # Handle duplicate dependencies without url | ||
105 | if not resolved: | ||
106 | return | ||
107 | |||
88 | localfile = npm_localfile(name, version) | 108 | localfile = npm_localfile(name, version) |
89 | 109 | ||
90 | uri = URI(resolved) | 110 | uri = URI(resolved) |
@@ -109,7 +129,7 @@ class NpmShrinkWrap(FetchMethod): | |||
109 | 129 | ||
110 | # Handle http tarball sources | 130 | # Handle http tarball sources |
111 | elif version.startswith("http") and integrity: | 131 | elif version.startswith("http") and integrity: |
112 | localfile = os.path.join("npm2", os.path.basename(version)) | 132 | localfile = npm_localfile(os.path.basename(version)) |
113 | 133 | ||
114 | uri = URI(version) | 134 | uri = URI(version) |
115 | uri.params["downloadfilename"] = localfile | 135 | uri.params["downloadfilename"] = localfile |
@@ -121,8 +141,28 @@ class NpmShrinkWrap(FetchMethod): | |||
121 | 141 | ||
122 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) | 142 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) |
123 | 143 | ||
144 | # Handle local tarball and link sources | ||
145 | elif version.startswith("file"): | ||
146 | localpath = version[5:] | ||
147 | if not version.endswith(".tgz"): | ||
148 | unpack = False | ||
149 | |||
124 | # Handle git sources | 150 | # Handle git sources |
125 | elif version.startswith("git"): | 151 | elif version.startswith(("git", "bitbucket","gist")) or ( |
152 | not version.endswith((".tgz", ".tar", ".tar.gz")) | ||
153 | and not version.startswith((".", "@", "/")) | ||
154 | and "/" in version | ||
155 | ): | ||
156 | if version.startswith("github:"): | ||
157 | version = "git+https://github.com/" + version[len("github:"):] | ||
158 | elif version.startswith("gist:"): | ||
159 | version = "git+https://gist.github.com/" + version[len("gist:"):] | ||
160 | elif version.startswith("bitbucket:"): | ||
161 | version = "git+https://bitbucket.org/" + version[len("bitbucket:"):] | ||
162 | elif version.startswith("gitlab:"): | ||
163 | version = "git+https://gitlab.com/" + version[len("gitlab:"):] | ||
164 | elif not version.startswith(("git+","git:")): | ||
165 | version = "git+https://github.com/" + version | ||
126 | regex = re.compile(r""" | 166 | regex = re.compile(r""" |
127 | ^ | 167 | ^ |
128 | git\+ | 168 | git\+ |
@@ -148,15 +188,17 @@ class NpmShrinkWrap(FetchMethod): | |||
148 | 188 | ||
149 | url = str(uri) | 189 | url = str(uri) |
150 | 190 | ||
151 | # local tarball sources and local link sources are unsupported | ||
152 | else: | 191 | else: |
153 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) | 192 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) |
154 | 193 | ||
194 | # name is needed by unpack tracer for module mapping | ||
155 | ud.deps.append({ | 195 | ud.deps.append({ |
196 | "name": name, | ||
156 | "url": url, | 197 | "url": url, |
157 | "localpath": localpath, | 198 | "localpath": localpath, |
158 | "extrapaths": extrapaths, | 199 | "extrapaths": extrapaths, |
159 | "destsuffix": destsuffix, | 200 | "destsuffix": destsuffix, |
201 | "unpack": unpack, | ||
160 | }) | 202 | }) |
161 | 203 | ||
162 | try: | 204 | try: |
@@ -177,17 +219,23 @@ class NpmShrinkWrap(FetchMethod): | |||
177 | # This fetcher resolves multiple URIs from a shrinkwrap file and then | 219 | # This fetcher resolves multiple URIs from a shrinkwrap file and then |
178 | # forwards it to a proxy fetcher. The management of the donestamp file, | 220 | # forwards it to a proxy fetcher. The management of the donestamp file, |
179 | # the lockfile and the checksums are forwarded to the proxy fetcher. | 221 | # the lockfile and the checksums are forwarded to the proxy fetcher. |
180 | ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) | 222 | shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]] |
223 | if shrinkwrap_urls: | ||
224 | ud.proxy = Fetch(shrinkwrap_urls, data) | ||
181 | ud.needdonestamp = False | 225 | ud.needdonestamp = False |
182 | 226 | ||
183 | @staticmethod | 227 | @staticmethod |
184 | def _foreach_proxy_method(ud, handle): | 228 | def _foreach_proxy_method(ud, handle): |
185 | returns = [] | 229 | returns = [] |
186 | for proxy_url in ud.proxy.urls: | 230 | #Check if there are dependencies before try to fetch them |
187 | proxy_ud = ud.proxy.ud[proxy_url] | 231 | if len(ud.deps) > 0: |
188 | proxy_d = ud.proxy.d | 232 | for proxy_url in ud.proxy.urls: |
189 | proxy_ud.setup_localpath(proxy_d) | 233 | proxy_ud = ud.proxy.ud[proxy_url] |
190 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | 234 | proxy_d = ud.proxy.d |
235 | proxy_ud.setup_localpath(proxy_d) | ||
236 | lf = lockfile(proxy_ud.lockfile) | ||
237 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | ||
238 | unlockfile(lf) | ||
191 | return returns | 239 | return returns |
192 | 240 | ||
193 | def verify_donestamp(self, ud, d): | 241 | def verify_donestamp(self, ud, d): |
@@ -220,10 +268,11 @@ class NpmShrinkWrap(FetchMethod): | |||
220 | 268 | ||
221 | def unpack(self, ud, rootdir, d): | 269 | def unpack(self, ud, rootdir, d): |
222 | """Unpack the downloaded dependencies""" | 270 | """Unpack the downloaded dependencies""" |
223 | destdir = d.getVar("S") | 271 | destdir = rootdir |
224 | destsuffix = ud.parm.get("destsuffix") | 272 | destsuffix = ud.parm.get("destsuffix") |
225 | if destsuffix: | 273 | if destsuffix: |
226 | destdir = os.path.join(rootdir, destsuffix) | 274 | destdir = os.path.join(rootdir, destsuffix) |
275 | ud.unpack_tracer.unpack("npm-shrinkwrap", destdir) | ||
227 | 276 | ||
228 | bb.utils.mkdirhier(destdir) | 277 | bb.utils.mkdirhier(destdir) |
229 | bb.utils.copyfile(ud.shrinkwrap_file, | 278 | bb.utils.copyfile(ud.shrinkwrap_file, |
@@ -237,7 +286,16 @@ class NpmShrinkWrap(FetchMethod): | |||
237 | 286 | ||
238 | for dep in manual: | 287 | for dep in manual: |
239 | depdestdir = os.path.join(destdir, dep["destsuffix"]) | 288 | depdestdir = os.path.join(destdir, dep["destsuffix"]) |
240 | npm_unpack(dep["localpath"], depdestdir, d) | 289 | if dep["url"]: |
290 | npm_unpack(dep["localpath"], depdestdir, d) | ||
291 | else: | ||
292 | depsrcdir= os.path.join(destdir, dep["localpath"]) | ||
293 | if dep["unpack"]: | ||
294 | npm_unpack(depsrcdir, depdestdir, d) | ||
295 | else: | ||
296 | bb.utils.mkdirhier(depdestdir) | ||
297 | cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) | ||
298 | runfetchcmd(cmd, d, workdir=depdestdir) | ||
241 | 299 | ||
242 | def clean(self, ud, d): | 300 | def clean(self, ud, d): |
243 | """Clean any existing full or partial download""" | 301 | """Clean any existing full or partial download""" |
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py index d9ce44390c..495ac8a30a 100644 --- a/bitbake/lib/bb/fetch2/osc.py +++ b/bitbake/lib/bb/fetch2/osc.py | |||
@@ -1,4 +1,6 @@ | |||
1 | # | 1 | # |
2 | # Copyright BitBake Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
3 | # | 5 | # |
4 | """ | 6 | """ |
@@ -9,6 +11,7 @@ Based on the svn "Fetch" implementation. | |||
9 | 11 | ||
10 | import logging | 12 | import logging |
11 | import os | 13 | import os |
14 | import re | ||
12 | import bb | 15 | import bb |
13 | from bb.fetch2 import FetchMethod | 16 | from bb.fetch2 import FetchMethod |
14 | from bb.fetch2 import FetchError | 17 | from bb.fetch2 import FetchError |
@@ -36,6 +39,7 @@ class Osc(FetchMethod): | |||
36 | # Create paths to osc checkouts | 39 | # Create paths to osc checkouts |
37 | oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc") | 40 | oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc") |
38 | relpath = self._strip_leading_slashes(ud.path) | 41 | relpath = self._strip_leading_slashes(ud.path) |
42 | ud.oscdir = oscdir | ||
39 | ud.pkgdir = os.path.join(oscdir, ud.host) | 43 | ud.pkgdir = os.path.join(oscdir, ud.host) |
40 | ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) | 44 | ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) |
41 | 45 | ||
@@ -43,13 +47,13 @@ class Osc(FetchMethod): | |||
43 | ud.revision = ud.parm['rev'] | 47 | ud.revision = ud.parm['rev'] |
44 | else: | 48 | else: |
45 | pv = d.getVar("PV", False) | 49 | pv = d.getVar("PV", False) |
46 | rev = bb.fetch2.srcrev_internal_helper(ud, d) | 50 | rev = bb.fetch2.srcrev_internal_helper(ud, d, '') |
47 | if rev: | 51 | if rev: |
48 | ud.revision = rev | 52 | ud.revision = rev |
49 | else: | 53 | else: |
50 | ud.revision = "" | 54 | ud.revision = "" |
51 | 55 | ||
52 | ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision)) | 56 | ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision)) |
53 | 57 | ||
54 | def _buildosccommand(self, ud, d, command): | 58 | def _buildosccommand(self, ud, d, command): |
55 | """ | 59 | """ |
@@ -59,26 +63,49 @@ class Osc(FetchMethod): | |||
59 | 63 | ||
60 | basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc" | 64 | basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc" |
61 | 65 | ||
62 | proto = ud.parm.get('protocol', 'ocs') | 66 | proto = ud.parm.get('protocol', 'https') |
63 | 67 | ||
64 | options = [] | 68 | options = [] |
65 | 69 | ||
66 | config = "-c %s" % self.generate_config(ud, d) | 70 | config = "-c %s" % self.generate_config(ud, d) |
67 | 71 | ||
68 | if ud.revision: | 72 | if getattr(ud, 'revision', ''): |
69 | options.append("-r %s" % ud.revision) | 73 | options.append("-r %s" % ud.revision) |
70 | 74 | ||
71 | coroot = self._strip_leading_slashes(ud.path) | 75 | coroot = self._strip_leading_slashes(ud.path) |
72 | 76 | ||
73 | if command == "fetch": | 77 | if command == "fetch": |
74 | osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) | 78 | osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options)) |
75 | elif command == "update": | 79 | elif command == "update": |
76 | osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) | 80 | osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options)) |
81 | elif command == "api_source": | ||
82 | osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module) | ||
77 | else: | 83 | else: |
78 | raise FetchError("Invalid osc command %s" % command, ud.url) | 84 | raise FetchError("Invalid osc command %s" % command, ud.url) |
79 | 85 | ||
80 | return osccmd | 86 | return osccmd |
81 | 87 | ||
88 | def _latest_revision(self, ud, d, name): | ||
89 | """ | ||
90 | Fetch latest revision for the given package | ||
91 | """ | ||
92 | api_source_cmd = self._buildosccommand(ud, d, "api_source") | ||
93 | |||
94 | output = runfetchcmd(api_source_cmd, d) | ||
95 | match = re.match(r'<directory ?.* rev="(\d+)".*>', output) | ||
96 | if match is None: | ||
97 | raise FetchError("Unable to parse osc response", ud.url) | ||
98 | return match.groups()[0] | ||
99 | |||
100 | def _revision_key(self, ud, d, name): | ||
101 | """ | ||
102 | Return a unique key for the url | ||
103 | """ | ||
104 | # Collapse adjacent slashes | ||
105 | slash_re = re.compile(r"/+") | ||
106 | rev = getattr(ud, 'revision', "latest") | ||
107 | return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev) | ||
108 | |||
82 | def download(self, ud, d): | 109 | def download(self, ud, d): |
83 | """ | 110 | """ |
84 | Fetch url | 111 | Fetch url |
@@ -86,7 +113,7 @@ class Osc(FetchMethod): | |||
86 | 113 | ||
87 | logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") | 114 | logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") |
88 | 115 | ||
89 | if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK): | 116 | if os.access(ud.moddir, os.R_OK): |
90 | oscupdatecmd = self._buildosccommand(ud, d, "update") | 117 | oscupdatecmd = self._buildosccommand(ud, d, "update") |
91 | logger.info("Update "+ ud.url) | 118 | logger.info("Update "+ ud.url) |
92 | # update sources there | 119 | # update sources there |
@@ -114,20 +141,23 @@ class Osc(FetchMethod): | |||
114 | Generate a .oscrc to be used for this run. | 141 | Generate a .oscrc to be used for this run. |
115 | """ | 142 | """ |
116 | 143 | ||
117 | config_path = os.path.join(d.getVar('OSCDIR'), "oscrc") | 144 | config_path = os.path.join(ud.oscdir, "oscrc") |
145 | if not os.path.exists(ud.oscdir): | ||
146 | bb.utils.mkdirhier(ud.oscdir) | ||
147 | |||
118 | if (os.path.exists(config_path)): | 148 | if (os.path.exists(config_path)): |
119 | os.remove(config_path) | 149 | os.remove(config_path) |
120 | 150 | ||
121 | f = open(config_path, 'w') | 151 | f = open(config_path, 'w') |
152 | proto = ud.parm.get('protocol', 'https') | ||
122 | f.write("[general]\n") | 153 | f.write("[general]\n") |
123 | f.write("apisrv = %s\n" % ud.host) | 154 | f.write("apiurl = %s://%s\n" % (proto, ud.host)) |
124 | f.write("scheme = http\n") | ||
125 | f.write("su-wrapper = su -c\n") | 155 | f.write("su-wrapper = su -c\n") |
126 | f.write("build-root = %s\n" % d.getVar('WORKDIR')) | 156 | f.write("build-root = %s\n" % d.getVar('WORKDIR')) |
127 | f.write("urllist = %s\n" % d.getVar("OSCURLLIST")) | 157 | f.write("urllist = %s\n" % d.getVar("OSCURLLIST")) |
128 | f.write("extra-pkgs = gzip\n") | 158 | f.write("extra-pkgs = gzip\n") |
129 | f.write("\n") | 159 | f.write("\n") |
130 | f.write("[%s]\n" % ud.host) | 160 | f.write("[%s://%s]\n" % (proto, ud.host)) |
131 | f.write("user = %s\n" % ud.parm["user"]) | 161 | f.write("user = %s\n" % ud.parm["user"]) |
132 | f.write("pass = %s\n" % ud.parm["pswd"]) | 162 | f.write("pass = %s\n" % ud.parm["pswd"]) |
133 | f.close() | 163 | f.close() |
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py index e2a41a4a12..3b6fa4b1ec 100644 --- a/bitbake/lib/bb/fetch2/perforce.py +++ b/bitbake/lib/bb/fetch2/perforce.py | |||
@@ -134,7 +134,7 @@ class Perforce(FetchMethod): | |||
134 | 134 | ||
135 | ud.setup_revisions(d) | 135 | ud.setup_revisions(d) |
136 | 136 | ||
137 | ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision)) | 137 | ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision)) |
138 | 138 | ||
139 | def _buildp4command(self, ud, d, command, depot_filename=None): | 139 | def _buildp4command(self, ud, d, command, depot_filename=None): |
140 | """ | 140 | """ |
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py index ffca73c8e4..6b8ffd5359 100644 --- a/bitbake/lib/bb/fetch2/s3.py +++ b/bitbake/lib/bb/fetch2/s3.py | |||
@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use. | |||
18 | import os | 18 | import os |
19 | import bb | 19 | import bb |
20 | import urllib.request, urllib.parse, urllib.error | 20 | import urllib.request, urllib.parse, urllib.error |
21 | import re | ||
21 | from bb.fetch2 import FetchMethod | 22 | from bb.fetch2 import FetchMethod |
22 | from bb.fetch2 import FetchError | 23 | from bb.fetch2 import FetchError |
23 | from bb.fetch2 import runfetchcmd | 24 | from bb.fetch2 import runfetchcmd |
24 | 25 | ||
26 | def convertToBytes(value, unit): | ||
27 | value = float(value) | ||
28 | if (unit == "KiB"): | ||
29 | value = value*1024.0; | ||
30 | elif (unit == "MiB"): | ||
31 | value = value*1024.0*1024.0; | ||
32 | elif (unit == "GiB"): | ||
33 | value = value*1024.0*1024.0*1024.0; | ||
34 | return value | ||
35 | |||
36 | class S3ProgressHandler(bb.progress.LineFilterProgressHandler): | ||
37 | """ | ||
38 | Extract progress information from s3 cp output, e.g.: | ||
39 | Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining | ||
40 | """ | ||
41 | def __init__(self, d): | ||
42 | super(S3ProgressHandler, self).__init__(d) | ||
43 | # Send an initial progress event so the bar gets shown | ||
44 | self._fire_progress(0) | ||
45 | |||
46 | def writeline(self, line): | ||
47 | percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line) | ||
48 | if percs: | ||
49 | completed = (percs[-1][0]) | ||
50 | completedUnit = (percs[-1][1]) | ||
51 | total = (percs[-1][2]) | ||
52 | totalUnit = (percs[-1][3]) | ||
53 | completed = convertToBytes(completed, completedUnit) | ||
54 | total = convertToBytes(total, totalUnit) | ||
55 | progress = (completed/total)*100.0 | ||
56 | rate = percs[-1][4] | ||
57 | self.update(progress, rate) | ||
58 | return False | ||
59 | return True | ||
60 | |||
61 | |||
25 | class S3(FetchMethod): | 62 | class S3(FetchMethod): |
26 | """Class to fetch urls via 'aws s3'""" | 63 | """Class to fetch urls via 'aws s3'""" |
27 | 64 | ||
@@ -52,7 +89,9 @@ class S3(FetchMethod): | |||
52 | 89 | ||
53 | cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) | 90 | cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) |
54 | bb.fetch2.check_network_access(d, cmd, ud.url) | 91 | bb.fetch2.check_network_access(d, cmd, ud.url) |
55 | runfetchcmd(cmd, d) | 92 | |
93 | progresshandler = S3ProgressHandler(d) | ||
94 | runfetchcmd(cmd, d, False, log=progresshandler) | ||
56 | 95 | ||
57 | # Additional sanity checks copied from the wget class (although there | 96 | # Additional sanity checks copied from the wget class (although there |
58 | # are no known issues which mean these are required, treat the aws cli | 97 | # are no known issues which mean these are required, treat the aws cli |
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py index f87f292e5d..7884cce949 100644 --- a/bitbake/lib/bb/fetch2/sftp.py +++ b/bitbake/lib/bb/fetch2/sftp.py | |||
@@ -103,7 +103,7 @@ class SFTP(FetchMethod): | |||
103 | if path[:3] == '/~/': | 103 | if path[:3] == '/~/': |
104 | path = path[3:] | 104 | path = path[3:] |
105 | 105 | ||
106 | remote = '%s%s:%s' % (user, urlo.hostname, path) | 106 | remote = '"%s%s:%s"' % (user, urlo.hostname, path) |
107 | 107 | ||
108 | cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) | 108 | cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) |
109 | 109 | ||
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py index 2c8557e1f8..0cbb2a6f25 100644 --- a/bitbake/lib/bb/fetch2/ssh.py +++ b/bitbake/lib/bb/fetch2/ssh.py | |||
@@ -32,6 +32,7 @@ IETF secsh internet draft: | |||
32 | 32 | ||
33 | import re, os | 33 | import re, os |
34 | from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd | 34 | from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd |
35 | import urllib | ||
35 | 36 | ||
36 | 37 | ||
37 | __pattern__ = re.compile(r''' | 38 | __pattern__ = re.compile(r''' |
@@ -40,9 +41,9 @@ __pattern__ = re.compile(r''' | |||
40 | ( # Optional username/password block | 41 | ( # Optional username/password block |
41 | (?P<user>\S+) # username | 42 | (?P<user>\S+) # username |
42 | (:(?P<pass>\S+))? # colon followed by the password (optional) | 43 | (:(?P<pass>\S+))? # colon followed by the password (optional) |
43 | )? | ||
44 | (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) | 44 | (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) |
45 | @ | 45 | @ |
46 | )? | ||
46 | (?P<host>\S+?) # non-greedy match of the host | 47 | (?P<host>\S+?) # non-greedy match of the host |
47 | (:(?P<port>[0-9]+))? # colon followed by the port (optional) | 48 | (:(?P<port>[0-9]+))? # colon followed by the port (optional) |
48 | / | 49 | / |
@@ -70,6 +71,7 @@ class SSH(FetchMethod): | |||
70 | "git:// prefix with protocol=ssh", urldata.url) | 71 | "git:// prefix with protocol=ssh", urldata.url) |
71 | m = __pattern__.match(urldata.url) | 72 | m = __pattern__.match(urldata.url) |
72 | path = m.group('path') | 73 | path = m.group('path') |
74 | path = urllib.parse.unquote(path) | ||
73 | host = m.group('host') | 75 | host = m.group('host') |
74 | urldata.localpath = os.path.join(d.getVar('DL_DIR'), | 76 | urldata.localpath = os.path.join(d.getVar('DL_DIR'), |
75 | os.path.basename(os.path.normpath(path))) | 77 | os.path.basename(os.path.normpath(path))) |
@@ -96,6 +98,11 @@ class SSH(FetchMethod): | |||
96 | fr += '@%s' % host | 98 | fr += '@%s' % host |
97 | else: | 99 | else: |
98 | fr = host | 100 | fr = host |
101 | |||
102 | if path[0] != '~': | ||
103 | path = '/%s' % path | ||
104 | path = urllib.parse.unquote(path) | ||
105 | |||
99 | fr += ':%s' % path | 106 | fr += ':%s' % path |
100 | 107 | ||
101 | cmd = 'scp -B -r %s %s %s/' % ( | 108 | cmd = 'scp -B -r %s %s %s/' % ( |
@@ -108,3 +115,41 @@ class SSH(FetchMethod): | |||
108 | 115 | ||
109 | runfetchcmd(cmd, d) | 116 | runfetchcmd(cmd, d) |
110 | 117 | ||
118 | def checkstatus(self, fetch, urldata, d): | ||
119 | """ | ||
120 | Check the status of the url | ||
121 | """ | ||
122 | m = __pattern__.match(urldata.url) | ||
123 | path = m.group('path') | ||
124 | host = m.group('host') | ||
125 | port = m.group('port') | ||
126 | user = m.group('user') | ||
127 | password = m.group('pass') | ||
128 | |||
129 | if port: | ||
130 | portarg = '-P %s' % port | ||
131 | else: | ||
132 | portarg = '' | ||
133 | |||
134 | if user: | ||
135 | fr = user | ||
136 | if password: | ||
137 | fr += ':%s' % password | ||
138 | fr += '@%s' % host | ||
139 | else: | ||
140 | fr = host | ||
141 | |||
142 | if path[0] != '~': | ||
143 | path = '/%s' % path | ||
144 | path = urllib.parse.unquote(path) | ||
145 | |||
146 | cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % ( | ||
147 | portarg, | ||
148 | fr, | ||
149 | path | ||
150 | ) | ||
151 | |||
152 | check_network_access(d, cmd, urldata.url) | ||
153 | runfetchcmd(cmd, d) | ||
154 | |||
155 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py index 8856ef1c62..0852108e7d 100644 --- a/bitbake/lib/bb/fetch2/svn.py +++ b/bitbake/lib/bb/fetch2/svn.py | |||
@@ -57,7 +57,12 @@ class Svn(FetchMethod): | |||
57 | if 'rev' in ud.parm: | 57 | if 'rev' in ud.parm: |
58 | ud.revision = ud.parm['rev'] | 58 | ud.revision = ud.parm['rev'] |
59 | 59 | ||
60 | ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision)) | 60 | # Whether to use the @REV peg-revision syntax in the svn command or not |
61 | ud.pegrevision = True | ||
62 | if 'nopegrevision' in ud.parm: | ||
63 | ud.pegrevision = False | ||
64 | |||
65 | ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision])) | ||
61 | 66 | ||
62 | def _buildsvncommand(self, ud, d, command): | 67 | def _buildsvncommand(self, ud, d, command): |
63 | """ | 68 | """ |
@@ -86,7 +91,7 @@ class Svn(FetchMethod): | |||
86 | if command == "info": | 91 | if command == "info": |
87 | svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) | 92 | svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) |
88 | elif command == "log1": | 93 | elif command == "log1": |
89 | svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) | 94 | svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) |
90 | else: | 95 | else: |
91 | suffix = "" | 96 | suffix = "" |
92 | 97 | ||
@@ -98,7 +103,8 @@ class Svn(FetchMethod): | |||
98 | 103 | ||
99 | if ud.revision: | 104 | if ud.revision: |
100 | options.append("-r %s" % ud.revision) | 105 | options.append("-r %s" % ud.revision) |
101 | suffix = "@%s" % (ud.revision) | 106 | if ud.pegrevision: |
107 | suffix = "@%s" % (ud.revision) | ||
102 | 108 | ||
103 | if command == "fetch": | 109 | if command == "fetch": |
104 | transportuser = ud.parm.get("transportuser", "") | 110 | transportuser = ud.parm.get("transportuser", "") |
@@ -204,3 +210,6 @@ class Svn(FetchMethod): | |||
204 | 210 | ||
205 | def _build_revision(self, ud, d): | 211 | def _build_revision(self, ud, d): |
206 | return ud.revision | 212 | return ud.revision |
213 | |||
214 | def supports_checksum(self, urldata): | ||
215 | return False | ||
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index 6d82f3af07..d76b1d0d38 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py | |||
@@ -26,7 +26,6 @@ from bb.fetch2 import FetchMethod | |||
26 | from bb.fetch2 import FetchError | 26 | from bb.fetch2 import FetchError |
27 | from bb.fetch2 import logger | 27 | from bb.fetch2 import logger |
28 | from bb.fetch2 import runfetchcmd | 28 | from bb.fetch2 import runfetchcmd |
29 | from bb.utils import export_proxies | ||
30 | from bs4 import BeautifulSoup | 29 | from bs4 import BeautifulSoup |
31 | from bs4 import SoupStrainer | 30 | from bs4 import SoupStrainer |
32 | 31 | ||
@@ -52,18 +51,24 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler): | |||
52 | 51 | ||
53 | 52 | ||
54 | class Wget(FetchMethod): | 53 | class Wget(FetchMethod): |
54 | """Class to fetch urls via 'wget'""" | ||
55 | 55 | ||
56 | # CDNs like CloudFlare may do a 'browser integrity test' which can fail | 56 | # CDNs like CloudFlare may do a 'browser integrity test' which can fail |
57 | # with the standard wget/urllib User-Agent, so pretend to be a modern | 57 | # with the standard wget/urllib User-Agent, so pretend to be a modern |
58 | # browser. | 58 | # browser. |
59 | user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" | 59 | user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" |
60 | 60 | ||
61 | """Class to fetch urls via 'wget'""" | 61 | def check_certs(self, d): |
62 | """ | ||
63 | Should certificates be checked? | ||
64 | """ | ||
65 | return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0" | ||
66 | |||
62 | def supports(self, ud, d): | 67 | def supports(self, ud, d): |
63 | """ | 68 | """ |
64 | Check to see if a given url can be fetched with wget. | 69 | Check to see if a given url can be fetched with wget. |
65 | """ | 70 | """ |
66 | return ud.type in ['http', 'https', 'ftp'] | 71 | return ud.type in ['http', 'https', 'ftp', 'ftps'] |
67 | 72 | ||
68 | def recommends_checksum(self, urldata): | 73 | def recommends_checksum(self, urldata): |
69 | return True | 74 | return True |
@@ -82,7 +87,13 @@ class Wget(FetchMethod): | |||
82 | if not ud.localfile: | 87 | if not ud.localfile: |
83 | ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) | 88 | ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) |
84 | 89 | ||
85 | self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" | 90 | self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30" |
91 | |||
92 | if ud.type == 'ftp' or ud.type == 'ftps': | ||
93 | self.basecmd += " --passive-ftp" | ||
94 | |||
95 | if not self.check_certs(d): | ||
96 | self.basecmd += " --no-check-certificate" | ||
86 | 97 | ||
87 | def _runwget(self, ud, d, command, quiet, workdir=None): | 98 | def _runwget(self, ud, d, command, quiet, workdir=None): |
88 | 99 | ||
@@ -97,13 +108,22 @@ class Wget(FetchMethod): | |||
97 | 108 | ||
98 | fetchcmd = self.basecmd | 109 | fetchcmd = self.basecmd |
99 | 110 | ||
100 | if 'downloadfilename' in ud.parm: | 111 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp" |
101 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | 112 | bb.utils.mkdirhier(os.path.dirname(localpath)) |
102 | bb.utils.mkdirhier(os.path.dirname(localpath)) | 113 | fetchcmd += " -O %s" % shlex.quote(localpath) |
103 | fetchcmd += " -O %s" % shlex.quote(localpath) | ||
104 | 114 | ||
105 | if ud.user and ud.pswd: | 115 | if ud.user and ud.pswd: |
106 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | 116 | fetchcmd += " --auth-no-challenge" |
117 | if ud.parm.get("redirectauth", "1") == "1": | ||
118 | # An undocumented feature of wget is that if the | ||
119 | # username/password are specified on the URI, wget will only | ||
120 | # send the Authorization header to the first host and not to | ||
121 | # any hosts that it is redirected to. With the increasing | ||
122 | # usage of temporary AWS URLs, this difference now matters as | ||
123 | # AWS will reject any request that has authentication both in | ||
124 | # the query parameters (from the redirect) and in the | ||
125 | # Authorization header. | ||
126 | fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd) | ||
107 | 127 | ||
108 | uri = ud.url.split(";")[0] | 128 | uri = ud.url.split(";")[0] |
109 | if os.path.exists(ud.localpath): | 129 | if os.path.exists(ud.localpath): |
@@ -116,13 +136,22 @@ class Wget(FetchMethod): | |||
116 | 136 | ||
117 | # Sanity check since wget can pretend it succeed when it didn't | 137 | # Sanity check since wget can pretend it succeed when it didn't |
118 | # Also, this used to happen if sourceforge sent us to the mirror page | 138 | # Also, this used to happen if sourceforge sent us to the mirror page |
119 | if not os.path.exists(ud.localpath): | 139 | if not os.path.exists(localpath): |
120 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) | 140 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri) |
121 | 141 | ||
122 | if os.path.getsize(ud.localpath) == 0: | 142 | if os.path.getsize(localpath) == 0: |
123 | os.remove(ud.localpath) | 143 | os.remove(localpath) |
124 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) | 144 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) |
125 | 145 | ||
146 | # Try and verify any checksum now, meaning if it isn't correct, we don't remove the | ||
147 | # original file, which might be a race (imagine two recipes referencing the same | ||
148 | # source, one with an incorrect checksum) | ||
149 | bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False) | ||
150 | |||
151 | # Remove the ".tmp" and move the file into position atomically | ||
152 | # Our lock prevents multiple writers but mirroring code may grab incomplete files | ||
153 | os.rename(localpath, localpath[:-4]) | ||
154 | |||
126 | return True | 155 | return True |
127 | 156 | ||
128 | def checkstatus(self, fetch, ud, d, try_again=True): | 157 | def checkstatus(self, fetch, ud, d, try_again=True): |
@@ -209,7 +238,7 @@ class Wget(FetchMethod): | |||
209 | # We let the request fail and expect it to be | 238 | # We let the request fail and expect it to be |
210 | # tried once more ("try_again" in check_status()), | 239 | # tried once more ("try_again" in check_status()), |
211 | # with the dead connection removed from the cache. | 240 | # with the dead connection removed from the cache. |
212 | # If it still fails, we give up, which can happend for bad | 241 | # If it still fails, we give up, which can happen for bad |
213 | # HTTP proxy settings. | 242 | # HTTP proxy settings. |
214 | fetch.connection_cache.remove_connection(h.host, h.port) | 243 | fetch.connection_cache.remove_connection(h.host, h.port) |
215 | raise urllib.error.URLError(err) | 244 | raise urllib.error.URLError(err) |
@@ -282,64 +311,76 @@ class Wget(FetchMethod): | |||
282 | newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) | 311 | newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) |
283 | newreq.get_method = req.get_method | 312 | newreq.get_method = req.get_method |
284 | return newreq | 313 | return newreq |
285 | exported_proxies = export_proxies(d) | ||
286 | |||
287 | handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] | ||
288 | if exported_proxies: | ||
289 | handlers.append(urllib.request.ProxyHandler()) | ||
290 | handlers.append(CacheHTTPHandler()) | ||
291 | # Since Python 2.7.9 ssl cert validation is enabled by default | ||
292 | # see PEP-0476, this causes verification errors on some https servers | ||
293 | # so disable by default. | ||
294 | import ssl | ||
295 | if hasattr(ssl, '_create_unverified_context'): | ||
296 | handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) | ||
297 | opener = urllib.request.build_opener(*handlers) | ||
298 | |||
299 | try: | ||
300 | uri = ud.url.split(";")[0] | ||
301 | r = urllib.request.Request(uri) | ||
302 | r.get_method = lambda: "HEAD" | ||
303 | # Some servers (FusionForge, as used on Alioth) require that the | ||
304 | # optional Accept header is set. | ||
305 | r.add_header("Accept", "*/*") | ||
306 | r.add_header("User-Agent", self.user_agent) | ||
307 | def add_basic_auth(login_str, request): | ||
308 | '''Adds Basic auth to http request, pass in login:password as string''' | ||
309 | import base64 | ||
310 | encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") | ||
311 | authheader = "Basic %s" % encodeuser | ||
312 | r.add_header("Authorization", authheader) | ||
313 | |||
314 | if ud.user and ud.pswd: | ||
315 | add_basic_auth(ud.user + ':' + ud.pswd, r) | ||
316 | 314 | ||
317 | try: | 315 | # We need to update the environment here as both the proxy and HTTPS |
318 | import netrc | 316 | # handlers need variables set. The proxy needs http_proxy and friends to |
319 | n = netrc.netrc() | 317 | # be set, and HTTPSHandler ends up calling into openssl to load the |
320 | login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) | 318 | # certificates. In buildtools configurations this will be looking at the |
321 | add_basic_auth("%s:%s" % (login, password), r) | 319 | # wrong place for certificates by default: we set SSL_CERT_FILE to the |
322 | except (TypeError, ImportError, IOError, netrc.NetrcParseError): | 320 | # right location in the buildtools environment script but as BitBake |
323 | pass | 321 | # prunes prunes the environment this is lost. When binaries are executed |
324 | 322 | # runfetchcmd ensures these values are in the environment, but this is | |
325 | with opener.open(r) as response: | 323 | # pure Python so we need to update the environment. |
326 | pass | 324 | # |
327 | except urllib.error.URLError as e: | 325 | # Avoid tramping the environment too much by using bb.utils.environment |
328 | if try_again: | 326 | # to scope the changes to the build_opener request, which is when the |
329 | logger.debug2("checkstatus: trying again") | 327 | # environment lookups happen. |
330 | return self.checkstatus(fetch, ud, d, False) | 328 | newenv = bb.fetch2.get_fetcher_environment(d) |
329 | |||
330 | with bb.utils.environment(**newenv): | ||
331 | import ssl | ||
332 | |||
333 | if self.check_certs(d): | ||
334 | context = ssl.create_default_context() | ||
331 | else: | 335 | else: |
332 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | 336 | context = ssl._create_unverified_context() |
333 | logger.debug2("checkstatus() urlopen failed: %s" % e) | 337 | |
334 | return False | 338 | handlers = [FixedHTTPRedirectHandler, |
335 | except ConnectionResetError as e: | 339 | HTTPMethodFallback, |
336 | if try_again: | 340 | urllib.request.ProxyHandler(), |
337 | logger.debug2("checkstatus: trying again") | 341 | CacheHTTPHandler(), |
338 | return self.checkstatus(fetch, ud, d, False) | 342 | urllib.request.HTTPSHandler(context=context)] |
339 | else: | 343 | opener = urllib.request.build_opener(*handlers) |
340 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | 344 | |
341 | logger.debug2("checkstatus() urlopen failed: %s" % e) | 345 | try: |
342 | return False | 346 | uri_base = ud.url.split(";")[0] |
347 | uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path) | ||
348 | r = urllib.request.Request(uri) | ||
349 | r.get_method = lambda: "HEAD" | ||
350 | # Some servers (FusionForge, as used on Alioth) require that the | ||
351 | # optional Accept header is set. | ||
352 | r.add_header("Accept", "*/*") | ||
353 | r.add_header("User-Agent", self.user_agent) | ||
354 | def add_basic_auth(login_str, request): | ||
355 | '''Adds Basic auth to http request, pass in login:password as string''' | ||
356 | import base64 | ||
357 | encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") | ||
358 | authheader = "Basic %s" % encodeuser | ||
359 | r.add_header("Authorization", authheader) | ||
360 | |||
361 | if ud.user and ud.pswd: | ||
362 | add_basic_auth(ud.user + ':' + ud.pswd, r) | ||
363 | |||
364 | try: | ||
365 | import netrc | ||
366 | auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname) | ||
367 | if auth_data: | ||
368 | login, _, password = auth_data | ||
369 | add_basic_auth("%s:%s" % (login, password), r) | ||
370 | except (FileNotFoundError, netrc.NetrcParseError): | ||
371 | pass | ||
372 | |||
373 | with opener.open(r, timeout=30) as response: | ||
374 | pass | ||
375 | except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e: | ||
376 | if try_again: | ||
377 | logger.debug2("checkstatus: trying again") | ||
378 | return self.checkstatus(fetch, ud, d, False) | ||
379 | else: | ||
380 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | ||
381 | logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e)) | ||
382 | return False | ||
383 | |||
343 | return True | 384 | return True |
344 | 385 | ||
345 | def _parse_path(self, regex, s): | 386 | def _parse_path(self, regex, s): |
@@ -472,7 +513,7 @@ class Wget(FetchMethod): | |||
472 | version_dir = ['', '', ''] | 513 | version_dir = ['', '', ''] |
473 | version = ['', '', ''] | 514 | version = ['', '', ''] |
474 | 515 | ||
475 | dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") | 516 | dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))") |
476 | s = dirver_regex.search(dirver) | 517 | s = dirver_regex.search(dirver) |
477 | if s: | 518 | if s: |
478 | version_dir[1] = s.group('ver') | 519 | version_dir[1] = s.group('ver') |
@@ -548,7 +589,7 @@ class Wget(FetchMethod): | |||
548 | 589 | ||
549 | # src.rpm extension was added only for rpm package. Can be removed if the rpm | 590 | # src.rpm extension was added only for rpm package. Can be removed if the rpm |
550 | # packaged will always be considered as having to be manually upgraded | 591 | # packaged will always be considered as having to be manually upgraded |
551 | psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" | 592 | psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)" |
552 | 593 | ||
553 | # match name, version and archive type of a package | 594 | # match name, version and archive type of a package |
554 | package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" | 595 | package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" |
@@ -599,10 +640,10 @@ class Wget(FetchMethod): | |||
599 | # search for version matches on folders inside the path, like: | 640 | # search for version matches on folders inside the path, like: |
600 | # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz | 641 | # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz |
601 | dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") | 642 | dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") |
602 | m = dirver_regex.search(path) | 643 | m = dirver_regex.findall(path) |
603 | if m: | 644 | if m: |
604 | pn = d.getVar('PN') | 645 | pn = d.getVar('PN') |
605 | dirver = m.group('dirver') | 646 | dirver = m[-1][0] |
606 | 647 | ||
607 | dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) | 648 | dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) |
608 | if not dirver_pn_regex.search(dirver): | 649 | if not dirver_pn_regex.search(dirver): |