diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r-- | bitbake/lib/bb/fetch2/README | 57 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 561 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/az.py | 98 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/clearcase.py | 6 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/crate.py | 150 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gcp.py | 102 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 669 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gitsm.py | 159 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gomod.py | 273 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/hg.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/local.py | 25 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npm.py | 83 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npmsw.py | 112 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/osc.py | 52 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/perforce.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/s3.py | 43 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/sftp.py | 4 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/ssh.py | 50 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/svn.py | 15 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/wget.py | 263 |
20 files changed, 2051 insertions, 674 deletions
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README new file mode 100644 index 0000000000..67b787ef47 --- /dev/null +++ b/bitbake/lib/bb/fetch2/README | |||
@@ -0,0 +1,57 @@ | |||
1 | There are expectations of users of the fetcher code. This file attempts to document | ||
2 | some of the constraints that are present. Some are obvious, some are less so. It is | ||
3 | documented in the context of how OE uses it but the API calls are generic. | ||
4 | |||
5 | a) network access for sources is only expected to happen in the do_fetch task. | ||
6 | This is not enforced or tested but is required so that we can: | ||
7 | |||
8 | i) audit the sources used (i.e. for license/manifest reasons) | ||
9 | ii) support offline builds with a suitable cache | ||
10 | iii) allow work to continue even with downtime upstream | ||
11 | iv) allow for changes upstream in incompatible ways | ||
12 | v) allow rebuilding of the software in X years time | ||
13 | |||
14 | b) network access is not expected in do_unpack task. | ||
15 | |||
16 | c) you can take DL_DIR and use it as a mirror for offline builds. | ||
17 | |||
18 | d) access to the network is only made when explicitly configured in recipes | ||
19 | (e.g. use of AUTOREV, or use of git tags which change revision). | ||
20 | |||
21 | e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it | ||
22 | will match in future exactly in a clean build with a new DL_DIR). | ||
23 | One specific pain point example are git tags. They can be replaced and change | ||
24 | so the git fetcher has to resolve them with the network. We use git revisions | ||
25 | where possible to avoid this and ensure determinism. | ||
26 | |||
27 | f) network access is expected to work with the standard linux proxy variables | ||
28 | so that access behind firewalls works (the fetcher sets these in the | ||
29 | environment but only in the do_fetch tasks). | ||
30 | |||
31 | g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV | ||
32 | git recipe might be ok but you can't expect to checkout a git tree. | ||
33 | |||
34 | h) we need to provide revision information during parsing such that a version | ||
35 | for the recipe can be constructed. | ||
36 | |||
37 | i) versions are expected to be able to increase in a way which sorts allowing | ||
38 | package feeds to operate (see PR server required for git revisions to sort). | ||
39 | |||
40 | j) API to query for possible version upgrades of a url is highly desireable to | ||
41 | allow our automated upgrage code to function (it is implied this does always | ||
42 | have network access). | ||
43 | |||
44 | k) Where fixes or changes to behaviour in the fetcher are made, we ask that | ||
45 | test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do | ||
46 | have fairly extensive test coverage of the fetcher as it is the only way | ||
47 | to track all of its corner cases, it still doesn't give entire coverage | ||
48 | though sadly. | ||
49 | |||
50 | l) If using tools during parse time, they will have to be in ASSUME_PROVIDED | ||
51 | in OE's context as we can't build git-native, then parse a recipe and use | ||
52 | git ls-remote. | ||
53 | |||
54 | Not all fetchers support all features, autorev is optional and doesn't make | ||
55 | sense for some. Upgrade detection means different things in different contexts | ||
56 | too. | ||
57 | |||
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 19169d780f..0ad987c596 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py | |||
@@ -23,17 +23,18 @@ import collections | |||
23 | import subprocess | 23 | import subprocess |
24 | import pickle | 24 | import pickle |
25 | import errno | 25 | import errno |
26 | import bb.persist_data, bb.utils | 26 | import bb.utils |
27 | import bb.checksum | 27 | import bb.checksum |
28 | import bb.process | 28 | import bb.process |
29 | import bb.event | 29 | import bb.event |
30 | 30 | ||
31 | __version__ = "2" | 31 | __version__ = "2" |
32 | _checksum_cache = bb.checksum.FileChecksumCache() | 32 | _checksum_cache = bb.checksum.FileChecksumCache() |
33 | _revisions_cache = bb.checksum.RevisionsCache() | ||
33 | 34 | ||
34 | logger = logging.getLogger("BitBake.Fetcher") | 35 | logger = logging.getLogger("BitBake.Fetcher") |
35 | 36 | ||
36 | CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] | 37 | CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ] |
37 | SHOWN_CHECKSUM_LIST = ["sha256"] | 38 | SHOWN_CHECKSUM_LIST = ["sha256"] |
38 | 39 | ||
39 | class BBFetchException(Exception): | 40 | class BBFetchException(Exception): |
@@ -113,7 +114,7 @@ class MissingParameterError(BBFetchException): | |||
113 | self.args = (missing, url) | 114 | self.args = (missing, url) |
114 | 115 | ||
115 | class ParameterError(BBFetchException): | 116 | class ParameterError(BBFetchException): |
116 | """Exception raised when a url cannot be proccessed due to invalid parameters.""" | 117 | """Exception raised when a url cannot be processed due to invalid parameters.""" |
117 | def __init__(self, message, url): | 118 | def __init__(self, message, url): |
118 | msg = "URL: '%s' has invalid parameters. %s" % (url, message) | 119 | msg = "URL: '%s' has invalid parameters. %s" % (url, message) |
119 | self.url = url | 120 | self.url = url |
@@ -182,7 +183,7 @@ class URI(object): | |||
182 | Some notes about relative URIs: while it's specified that | 183 | Some notes about relative URIs: while it's specified that |
183 | a URI beginning with <scheme>:// should either be directly | 184 | a URI beginning with <scheme>:// should either be directly |
184 | followed by a hostname or a /, the old URI handling of the | 185 | followed by a hostname or a /, the old URI handling of the |
185 | fetch2 library did not comform to this. Therefore, this URI | 186 | fetch2 library did not conform to this. Therefore, this URI |
186 | class has some kludges to make sure that URIs are parsed in | 187 | class has some kludges to make sure that URIs are parsed in |
187 | a way comforming to bitbake's current usage. This URI class | 188 | a way comforming to bitbake's current usage. This URI class |
188 | supports the following: | 189 | supports the following: |
@@ -199,7 +200,7 @@ class URI(object): | |||
199 | file://hostname/absolute/path.diff (would be IETF compliant) | 200 | file://hostname/absolute/path.diff (would be IETF compliant) |
200 | 201 | ||
201 | Note that the last case only applies to a list of | 202 | Note that the last case only applies to a list of |
202 | "whitelisted" schemes (currently only file://), that requires | 203 | explicitly allowed schemes (currently only file://), that requires |
203 | its URIs to not have a network location. | 204 | its URIs to not have a network location. |
204 | """ | 205 | """ |
205 | 206 | ||
@@ -237,7 +238,7 @@ class URI(object): | |||
237 | # to RFC compliant URL format. E.g.: | 238 | # to RFC compliant URL format. E.g.: |
238 | # file://foo.diff -> file:foo.diff | 239 | # file://foo.diff -> file:foo.diff |
239 | if urlp.scheme in self._netloc_forbidden: | 240 | if urlp.scheme in self._netloc_forbidden: |
240 | uri = re.sub("(?<=:)//(?!/)", "", uri, 1) | 241 | uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1) |
241 | reparse = 1 | 242 | reparse = 1 |
242 | 243 | ||
243 | if reparse: | 244 | if reparse: |
@@ -290,12 +291,12 @@ class URI(object): | |||
290 | 291 | ||
291 | def _param_str_split(self, string, elmdelim, kvdelim="="): | 292 | def _param_str_split(self, string, elmdelim, kvdelim="="): |
292 | ret = collections.OrderedDict() | 293 | ret = collections.OrderedDict() |
293 | for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: | 294 | for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]: |
294 | ret[k] = v | 295 | ret[k] = v |
295 | return ret | 296 | return ret |
296 | 297 | ||
297 | def _param_str_join(self, dict_, elmdelim, kvdelim="="): | 298 | def _param_str_join(self, dict_, elmdelim, kvdelim="="): |
298 | return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) | 299 | return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()]) |
299 | 300 | ||
300 | @property | 301 | @property |
301 | def hostport(self): | 302 | def hostport(self): |
@@ -352,6 +353,14 @@ def decodeurl(url): | |||
352 | user, password, parameters). | 353 | user, password, parameters). |
353 | """ | 354 | """ |
354 | 355 | ||
356 | uri = URI(url) | ||
357 | path = uri.path if uri.path else "/" | ||
358 | return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params | ||
359 | |||
360 | def decodemirrorurl(url): | ||
361 | """Decodes a mirror URL into the tokens (scheme, network location, path, | ||
362 | user, password, parameters). | ||
363 | """ | ||
355 | m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) | 364 | m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) |
356 | if not m: | 365 | if not m: |
357 | raise MalformedUrl(url) | 366 | raise MalformedUrl(url) |
@@ -370,6 +379,9 @@ def decodeurl(url): | |||
370 | elif type.lower() == 'file': | 379 | elif type.lower() == 'file': |
371 | host = "" | 380 | host = "" |
372 | path = location | 381 | path = location |
382 | if user: | ||
383 | path = user + '@' + path | ||
384 | user = "" | ||
373 | else: | 385 | else: |
374 | host = location | 386 | host = location |
375 | path = "/" | 387 | path = "/" |
@@ -388,7 +400,7 @@ def decodeurl(url): | |||
388 | if s: | 400 | if s: |
389 | if not '=' in s: | 401 | if not '=' in s: |
390 | raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) | 402 | raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) |
391 | s1, s2 = s.split('=') | 403 | s1, s2 = s.split('=', 1) |
392 | p[s1] = s2 | 404 | p[s1] = s2 |
393 | 405 | ||
394 | return type, host, urllib.parse.unquote(path), user, pswd, p | 406 | return type, host, urllib.parse.unquote(path), user, pswd, p |
@@ -402,34 +414,37 @@ def encodeurl(decoded): | |||
402 | 414 | ||
403 | if not type: | 415 | if not type: |
404 | raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) | 416 | raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) |
405 | url = '%s://' % type | 417 | uri = URI() |
418 | uri.scheme = type | ||
406 | if user and type != "file": | 419 | if user and type != "file": |
407 | url += "%s" % user | 420 | uri.username = user |
408 | if pswd: | 421 | if pswd: |
409 | url += ":%s" % pswd | 422 | uri.password = pswd |
410 | url += "@" | ||
411 | if host and type != "file": | 423 | if host and type != "file": |
412 | url += "%s" % host | 424 | uri.hostname = host |
413 | if path: | 425 | if path: |
414 | # Standardise path to ensure comparisons work | 426 | # Standardise path to ensure comparisons work |
415 | while '//' in path: | 427 | while '//' in path: |
416 | path = path.replace("//", "/") | 428 | path = path.replace("//", "/") |
417 | url += "%s" % urllib.parse.quote(path) | 429 | uri.path = path |
430 | if type == "file": | ||
431 | # Use old not IETF compliant style | ||
432 | uri.relative = False | ||
418 | if p: | 433 | if p: |
419 | for parm in p: | 434 | uri.params = p |
420 | url += ";%s=%s" % (parm, p[parm]) | ||
421 | 435 | ||
422 | return url | 436 | return str(uri) |
423 | 437 | ||
424 | def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | 438 | def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): |
425 | if not ud.url or not uri_find or not uri_replace: | 439 | if not ud.url or not uri_find or not uri_replace: |
426 | logger.error("uri_replace: passed an undefined value, not replacing") | 440 | logger.error("uri_replace: passed an undefined value, not replacing") |
427 | return None | 441 | return None |
428 | uri_decoded = list(decodeurl(ud.url)) | 442 | uri_decoded = list(decodemirrorurl(ud.url)) |
429 | uri_find_decoded = list(decodeurl(uri_find)) | 443 | uri_find_decoded = list(decodemirrorurl(uri_find)) |
430 | uri_replace_decoded = list(decodeurl(uri_replace)) | 444 | uri_replace_decoded = list(decodemirrorurl(uri_replace)) |
431 | logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) | 445 | logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) |
432 | result_decoded = ['', '', '', '', '', {}] | 446 | result_decoded = ['', '', '', '', '', {}] |
447 | # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params | ||
433 | for loc, i in enumerate(uri_find_decoded): | 448 | for loc, i in enumerate(uri_find_decoded): |
434 | result_decoded[loc] = uri_decoded[loc] | 449 | result_decoded[loc] = uri_decoded[loc] |
435 | regexp = i | 450 | regexp = i |
@@ -449,6 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
449 | for l in replacements: | 464 | for l in replacements: |
450 | uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) | 465 | uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) |
451 | result_decoded[loc][k] = uri_replace_decoded[loc][k] | 466 | result_decoded[loc][k] = uri_replace_decoded[loc][k] |
467 | elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]: | ||
468 | # User/password in the replacement is just a straight replacement | ||
469 | result_decoded[loc] = uri_replace_decoded[loc] | ||
452 | elif (re.match(regexp, uri_decoded[loc])): | 470 | elif (re.match(regexp, uri_decoded[loc])): |
453 | if not uri_replace_decoded[loc]: | 471 | if not uri_replace_decoded[loc]: |
454 | result_decoded[loc] = "" | 472 | result_decoded[loc] = "" |
@@ -456,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
456 | for k in replacements: | 474 | for k in replacements: |
457 | uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) | 475 | uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) |
458 | #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) | 476 | #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) |
459 | result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) | 477 | result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1) |
460 | if loc == 2: | 478 | if loc == 2: |
461 | # Handle path manipulations | 479 | # Handle path manipulations |
462 | basename = None | 480 | basename = None |
@@ -465,10 +483,18 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): | |||
465 | basename = os.path.basename(mirrortarball) | 483 | basename = os.path.basename(mirrortarball) |
466 | # Kill parameters, they make no sense for mirror tarballs | 484 | # Kill parameters, they make no sense for mirror tarballs |
467 | uri_decoded[5] = {} | 485 | uri_decoded[5] = {} |
486 | uri_find_decoded[5] = {} | ||
468 | elif ud.localpath and ud.method.supports_checksum(ud): | 487 | elif ud.localpath and ud.method.supports_checksum(ud): |
469 | basename = os.path.basename(ud.localpath) | 488 | basename = os.path.basename(ud.localpath) |
470 | if basename and not result_decoded[loc].endswith(basename): | 489 | if basename: |
471 | result_decoded[loc] = os.path.join(result_decoded[loc], basename) | 490 | uri_basename = os.path.basename(uri_decoded[loc]) |
491 | # Prefix with a slash as a sentinel in case | ||
492 | # result_decoded[loc] does not contain one. | ||
493 | path = "/" + result_decoded[loc] | ||
494 | if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): | ||
495 | result_decoded[loc] = path[1:-len(uri_basename)] + basename | ||
496 | elif not path.endswith("/" + basename): | ||
497 | result_decoded[loc] = os.path.join(path[1:], basename) | ||
472 | else: | 498 | else: |
473 | return None | 499 | return None |
474 | result = encodeurl(result_decoded) | 500 | result = encodeurl(result_decoded) |
@@ -481,18 +507,23 @@ methods = [] | |||
481 | urldata_cache = {} | 507 | urldata_cache = {} |
482 | saved_headrevs = {} | 508 | saved_headrevs = {} |
483 | 509 | ||
484 | def fetcher_init(d): | 510 | def fetcher_init(d, servercontext=True): |
485 | """ | 511 | """ |
486 | Called to initialize the fetchers once the configuration data is known. | 512 | Called to initialize the fetchers once the configuration data is known. |
487 | Calls before this must not hit the cache. | 513 | Calls before this must not hit the cache. |
488 | """ | 514 | """ |
489 | 515 | ||
490 | revs = bb.persist_data.persist('BB_URI_HEADREVS', d) | 516 | _checksum_cache.init_cache(d.getVar("BB_CACHEDIR")) |
517 | _revisions_cache.init_cache(d.getVar("BB_CACHEDIR")) | ||
518 | |||
519 | if not servercontext: | ||
520 | return | ||
521 | |||
491 | try: | 522 | try: |
492 | # fetcher_init is called multiple times, so make sure we only save the | 523 | # fetcher_init is called multiple times, so make sure we only save the |
493 | # revs the first time it is called. | 524 | # revs the first time it is called. |
494 | if not bb.fetch2.saved_headrevs: | 525 | if not bb.fetch2.saved_headrevs: |
495 | bb.fetch2.saved_headrevs = dict(revs) | 526 | bb.fetch2.saved_headrevs = _revisions_cache.get_revs() |
496 | except: | 527 | except: |
497 | pass | 528 | pass |
498 | 529 | ||
@@ -502,11 +533,10 @@ def fetcher_init(d): | |||
502 | logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) | 533 | logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) |
503 | elif srcrev_policy == "clear": | 534 | elif srcrev_policy == "clear": |
504 | logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) | 535 | logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) |
505 | revs.clear() | 536 | _revisions_cache.clear_cache() |
506 | else: | 537 | else: |
507 | raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) | 538 | raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) |
508 | 539 | ||
509 | _checksum_cache.init_cache(d) | ||
510 | 540 | ||
511 | for m in methods: | 541 | for m in methods: |
512 | if hasattr(m, "init"): | 542 | if hasattr(m, "init"): |
@@ -514,9 +544,11 @@ def fetcher_init(d): | |||
514 | 544 | ||
515 | def fetcher_parse_save(): | 545 | def fetcher_parse_save(): |
516 | _checksum_cache.save_extras() | 546 | _checksum_cache.save_extras() |
547 | _revisions_cache.save_extras() | ||
517 | 548 | ||
518 | def fetcher_parse_done(): | 549 | def fetcher_parse_done(): |
519 | _checksum_cache.save_merge() | 550 | _checksum_cache.save_merge() |
551 | _revisions_cache.save_merge() | ||
520 | 552 | ||
521 | def fetcher_compare_revisions(d): | 553 | def fetcher_compare_revisions(d): |
522 | """ | 554 | """ |
@@ -524,7 +556,7 @@ def fetcher_compare_revisions(d): | |||
524 | when bitbake was started and return true if they have changed. | 556 | when bitbake was started and return true if they have changed. |
525 | """ | 557 | """ |
526 | 558 | ||
527 | headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) | 559 | headrevs = _revisions_cache.get_revs() |
528 | return headrevs != bb.fetch2.saved_headrevs | 560 | return headrevs != bb.fetch2.saved_headrevs |
529 | 561 | ||
530 | def mirror_from_string(data): | 562 | def mirror_from_string(data): |
@@ -534,7 +566,7 @@ def mirror_from_string(data): | |||
534 | bb.warn('Invalid mirror data %s, should have paired members.' % data) | 566 | bb.warn('Invalid mirror data %s, should have paired members.' % data) |
535 | return list(zip(*[iter(mirrors)]*2)) | 567 | return list(zip(*[iter(mirrors)]*2)) |
536 | 568 | ||
537 | def verify_checksum(ud, d, precomputed={}): | 569 | def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True): |
538 | """ | 570 | """ |
539 | verify the MD5 and SHA256 checksum for downloaded src | 571 | verify the MD5 and SHA256 checksum for downloaded src |
540 | 572 | ||
@@ -548,20 +580,25 @@ def verify_checksum(ud, d, precomputed={}): | |||
548 | file against those in the recipe each time, rather than only after | 580 | file against those in the recipe each time, rather than only after |
549 | downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. | 581 | downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. |
550 | """ | 582 | """ |
551 | |||
552 | if ud.ignore_checksums or not ud.method.supports_checksum(ud): | 583 | if ud.ignore_checksums or not ud.method.supports_checksum(ud): |
553 | return {} | 584 | return {} |
554 | 585 | ||
586 | if localpath is None: | ||
587 | localpath = ud.localpath | ||
588 | |||
555 | def compute_checksum_info(checksum_id): | 589 | def compute_checksum_info(checksum_id): |
556 | checksum_name = getattr(ud, "%s_name" % checksum_id) | 590 | checksum_name = getattr(ud, "%s_name" % checksum_id) |
557 | 591 | ||
558 | if checksum_id in precomputed: | 592 | if checksum_id in precomputed: |
559 | checksum_data = precomputed[checksum_id] | 593 | checksum_data = precomputed[checksum_id] |
560 | else: | 594 | else: |
561 | checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) | 595 | checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath) |
562 | 596 | ||
563 | checksum_expected = getattr(ud, "%s_expected" % checksum_id) | 597 | checksum_expected = getattr(ud, "%s_expected" % checksum_id) |
564 | 598 | ||
599 | if checksum_expected == '': | ||
600 | checksum_expected = None | ||
601 | |||
565 | return { | 602 | return { |
566 | "id": checksum_id, | 603 | "id": checksum_id, |
567 | "name": checksum_name, | 604 | "name": checksum_name, |
@@ -581,17 +618,13 @@ def verify_checksum(ud, d, precomputed={}): | |||
581 | checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] | 618 | checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] |
582 | 619 | ||
583 | # If no checksum has been provided | 620 | # If no checksum has been provided |
584 | if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): | 621 | if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): |
585 | messages = [] | 622 | messages = [] |
586 | strict = d.getVar("BB_STRICT_CHECKSUM") or "0" | 623 | strict = d.getVar("BB_STRICT_CHECKSUM") or "0" |
587 | 624 | ||
588 | # If strict checking enabled and neither sum defined, raise error | 625 | # If strict checking enabled and neither sum defined, raise error |
589 | if strict == "1": | 626 | if strict == "1": |
590 | messages.append("No checksum specified for '%s', please add at " \ | 627 | raise NoChecksumError("\n".join(checksum_lines)) |
591 | "least one to the recipe:" % ud.localpath) | ||
592 | messages.extend(checksum_lines) | ||
593 | logger.error("\n".join(messages)) | ||
594 | raise NoChecksumError("Missing SRC_URI checksum", ud.url) | ||
595 | 628 | ||
596 | bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) | 629 | bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) |
597 | 630 | ||
@@ -612,8 +645,8 @@ def verify_checksum(ud, d, precomputed={}): | |||
612 | 645 | ||
613 | for ci in checksum_infos: | 646 | for ci in checksum_infos: |
614 | if ci["expected"] and ci["expected"] != ci["data"]: | 647 | if ci["expected"] and ci["expected"] != ci["data"]: |
615 | messages.append("File: '%s' has %s checksum %s when %s was " \ | 648 | messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ |
616 | "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) | 649 | "expected" % (localpath, ci["id"], ci["data"], ci["expected"])) |
617 | bad_checksum = ci["data"] | 650 | bad_checksum = ci["data"] |
618 | 651 | ||
619 | if bad_checksum: | 652 | if bad_checksum: |
@@ -731,13 +764,16 @@ def subprocess_setup(): | |||
731 | # SIGPIPE errors are known issues with gzip/bash | 764 | # SIGPIPE errors are known issues with gzip/bash |
732 | signal.signal(signal.SIGPIPE, signal.SIG_DFL) | 765 | signal.signal(signal.SIGPIPE, signal.SIG_DFL) |
733 | 766 | ||
734 | def get_autorev(d): | 767 | def mark_recipe_nocache(d): |
735 | # only not cache src rev in autorev case | ||
736 | if d.getVar('BB_SRCREV_POLICY') != "cache": | 768 | if d.getVar('BB_SRCREV_POLICY') != "cache": |
737 | d.setVar('BB_DONT_CACHE', '1') | 769 | d.setVar('BB_DONT_CACHE', '1') |
770 | |||
771 | def get_autorev(d): | ||
772 | mark_recipe_nocache(d) | ||
773 | d.setVar("__BBAUTOREV_SEEN", True) | ||
738 | return "AUTOINC" | 774 | return "AUTOINC" |
739 | 775 | ||
740 | def get_srcrev(d, method_name='sortable_revision'): | 776 | def _get_srcrev(d, method_name='sortable_revision'): |
741 | """ | 777 | """ |
742 | Return the revision string, usually for use in the version string (PV) of the current package | 778 | Return the revision string, usually for use in the version string (PV) of the current package |
743 | Most packages usually only have one SCM so we just pass on the call. | 779 | Most packages usually only have one SCM so we just pass on the call. |
@@ -751,23 +787,34 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
751 | that fetcher provides a method with the given name and the same signature as sortable_revision. | 787 | that fetcher provides a method with the given name and the same signature as sortable_revision. |
752 | """ | 788 | """ |
753 | 789 | ||
790 | d.setVar("__BBSRCREV_SEEN", "1") | ||
791 | recursion = d.getVar("__BBINSRCREV") | ||
792 | if recursion: | ||
793 | raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") | ||
794 | d.setVar("__BBINSRCREV", True) | ||
795 | |||
754 | scms = [] | 796 | scms = [] |
797 | revs = [] | ||
755 | fetcher = Fetch(d.getVar('SRC_URI').split(), d) | 798 | fetcher = Fetch(d.getVar('SRC_URI').split(), d) |
756 | urldata = fetcher.ud | 799 | urldata = fetcher.ud |
757 | for u in urldata: | 800 | for u in urldata: |
758 | if urldata[u].method.supports_srcrev(): | 801 | if urldata[u].method.supports_srcrev(): |
759 | scms.append(u) | 802 | scms.append(u) |
760 | 803 | ||
761 | if len(scms) == 0: | 804 | if not scms: |
762 | raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") | 805 | d.delVar("__BBINSRCREV") |
806 | return "", revs | ||
763 | 807 | ||
764 | if len(scms) == 1 and len(urldata[scms[0]].names) == 1: | 808 | |
765 | autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) | 809 | if len(scms) == 1: |
810 | autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name) | ||
811 | revs.append(rev) | ||
766 | if len(rev) > 10: | 812 | if len(rev) > 10: |
767 | rev = rev[:10] | 813 | rev = rev[:10] |
814 | d.delVar("__BBINSRCREV") | ||
768 | if autoinc: | 815 | if autoinc: |
769 | return "AUTOINC+" + rev | 816 | return "AUTOINC+" + rev, revs |
770 | return rev | 817 | return rev, revs |
771 | 818 | ||
772 | # | 819 | # |
773 | # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT | 820 | # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT |
@@ -781,12 +828,12 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
781 | seenautoinc = False | 828 | seenautoinc = False |
782 | for scm in scms: | 829 | for scm in scms: |
783 | ud = urldata[scm] | 830 | ud = urldata[scm] |
784 | for name in ud.names: | 831 | autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name) |
785 | autoinc, rev = getattr(ud.method, method_name)(ud, d, name) | 832 | revs.append(rev) |
786 | seenautoinc = seenautoinc or autoinc | 833 | seenautoinc = seenautoinc or autoinc |
787 | if len(rev) > 10: | 834 | if len(rev) > 10: |
788 | rev = rev[:10] | 835 | rev = rev[:10] |
789 | name_to_rev[name] = rev | 836 | name_to_rev[ud.name] = rev |
790 | # Replace names by revisions in the SRCREV_FORMAT string. The approach used | 837 | # Replace names by revisions in the SRCREV_FORMAT string. The approach used |
791 | # here can handle names being prefixes of other names and names appearing | 838 | # here can handle names being prefixes of other names and names appearing |
792 | # as substrings in revisions (in which case the name should not be | 839 | # as substrings in revisions (in which case the name should not be |
@@ -799,12 +846,71 @@ def get_srcrev(d, method_name='sortable_revision'): | |||
799 | if seenautoinc: | 846 | if seenautoinc: |
800 | format = "AUTOINC+" + format | 847 | format = "AUTOINC+" + format |
801 | 848 | ||
802 | return format | 849 | d.delVar("__BBINSRCREV") |
850 | return format, revs | ||
851 | |||
852 | def get_hashvalue(d, method_name='sortable_revision'): | ||
853 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
854 | return " ".join(revs) | ||
855 | |||
856 | def get_pkgv_string(d, method_name='sortable_revision'): | ||
857 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
858 | return pkgv | ||
859 | |||
860 | def get_srcrev(d, method_name='sortable_revision'): | ||
861 | pkgv, revs = _get_srcrev(d, method_name=method_name) | ||
862 | if not pkgv: | ||
863 | raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") | ||
864 | return pkgv | ||
803 | 865 | ||
804 | def localpath(url, d): | 866 | def localpath(url, d): |
805 | fetcher = bb.fetch2.Fetch([url], d) | 867 | fetcher = bb.fetch2.Fetch([url], d) |
806 | return fetcher.localpath(url) | 868 | return fetcher.localpath(url) |
807 | 869 | ||
870 | # Need to export PATH as binary could be in metadata paths | ||
871 | # rather than host provided | ||
872 | # Also include some other variables. | ||
873 | FETCH_EXPORT_VARS = ['HOME', 'PATH', | ||
874 | 'HTTP_PROXY', 'http_proxy', | ||
875 | 'HTTPS_PROXY', 'https_proxy', | ||
876 | 'FTP_PROXY', 'ftp_proxy', | ||
877 | 'FTPS_PROXY', 'ftps_proxy', | ||
878 | 'NO_PROXY', 'no_proxy', | ||
879 | 'ALL_PROXY', 'all_proxy', | ||
880 | 'GIT_PROXY_COMMAND', | ||
881 | 'GIT_SSH', | ||
882 | 'GIT_SSH_COMMAND', | ||
883 | 'GIT_SSL_CAINFO', | ||
884 | 'GIT_SMART_HTTP', | ||
885 | 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', | ||
886 | 'SOCKS5_USER', 'SOCKS5_PASSWD', | ||
887 | 'DBUS_SESSION_BUS_ADDRESS', | ||
888 | 'P4CONFIG', | ||
889 | 'SSL_CERT_FILE', | ||
890 | 'NODE_EXTRA_CA_CERTS', | ||
891 | 'AWS_PROFILE', | ||
892 | 'AWS_ACCESS_KEY_ID', | ||
893 | 'AWS_SECRET_ACCESS_KEY', | ||
894 | 'AWS_ROLE_ARN', | ||
895 | 'AWS_WEB_IDENTITY_TOKEN_FILE', | ||
896 | 'AWS_DEFAULT_REGION', | ||
897 | 'AWS_SESSION_TOKEN', | ||
898 | 'GIT_CACHE_PATH', | ||
899 | 'REMOTE_CONTAINERS_IPC', | ||
900 | 'GITHUB_TOKEN', | ||
901 | 'SSL_CERT_DIR'] | ||
902 | |||
903 | def get_fetcher_environment(d): | ||
904 | newenv = {} | ||
905 | origenv = d.getVar("BB_ORIGENV") | ||
906 | for name in bb.fetch2.FETCH_EXPORT_VARS: | ||
907 | value = d.getVar(name) | ||
908 | if not value and origenv: | ||
909 | value = origenv.getVar(name) | ||
910 | if value: | ||
911 | newenv[name] = value | ||
912 | return newenv | ||
913 | |||
808 | def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | 914 | def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): |
809 | """ | 915 | """ |
810 | Run cmd returning the command output | 916 | Run cmd returning the command output |
@@ -813,25 +919,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | |||
813 | Optionally remove the files/directories listed in cleanup upon failure | 919 | Optionally remove the files/directories listed in cleanup upon failure |
814 | """ | 920 | """ |
815 | 921 | ||
816 | # Need to export PATH as binary could be in metadata paths | 922 | exportvars = FETCH_EXPORT_VARS |
817 | # rather than host provided | ||
818 | # Also include some other variables. | ||
819 | # FIXME: Should really include all export varaiables? | ||
820 | exportvars = ['HOME', 'PATH', | ||
821 | 'HTTP_PROXY', 'http_proxy', | ||
822 | 'HTTPS_PROXY', 'https_proxy', | ||
823 | 'FTP_PROXY', 'ftp_proxy', | ||
824 | 'FTPS_PROXY', 'ftps_proxy', | ||
825 | 'NO_PROXY', 'no_proxy', | ||
826 | 'ALL_PROXY', 'all_proxy', | ||
827 | 'GIT_PROXY_COMMAND', | ||
828 | 'GIT_SSH', | ||
829 | 'GIT_SSL_CAINFO', | ||
830 | 'GIT_SMART_HTTP', | ||
831 | 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', | ||
832 | 'SOCKS5_USER', 'SOCKS5_PASSWD', | ||
833 | 'DBUS_SESSION_BUS_ADDRESS', | ||
834 | 'P4CONFIG'] | ||
835 | 923 | ||
836 | if not cleanup: | 924 | if not cleanup: |
837 | cleanup = [] | 925 | cleanup = [] |
@@ -868,14 +956,17 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): | |||
868 | (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) | 956 | (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) |
869 | success = True | 957 | success = True |
870 | except bb.process.NotFoundError as e: | 958 | except bb.process.NotFoundError as e: |
871 | error_message = "Fetch command %s" % (e.command) | 959 | error_message = "Fetch command %s not found" % (e.command) |
872 | except bb.process.ExecutionError as e: | 960 | except bb.process.ExecutionError as e: |
873 | if e.stdout: | 961 | if e.stdout: |
874 | output = "output:\n%s\n%s" % (e.stdout, e.stderr) | 962 | output = "output:\n%s\n%s" % (e.stdout, e.stderr) |
875 | elif e.stderr: | 963 | elif e.stderr: |
876 | output = "output:\n%s" % e.stderr | 964 | output = "output:\n%s" % e.stderr |
877 | else: | 965 | else: |
878 | output = "no output" | 966 | if log: |
967 | output = "see logfile for output" | ||
968 | else: | ||
969 | output = "no output" | ||
879 | error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) | 970 | error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) |
880 | except bb.process.CmdError as e: | 971 | except bb.process.CmdError as e: |
881 | error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) | 972 | error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) |
@@ -937,6 +1028,7 @@ def build_mirroruris(origud, mirrors, ld): | |||
937 | 1028 | ||
938 | try: | 1029 | try: |
939 | newud = FetchData(newuri, ld) | 1030 | newud = FetchData(newuri, ld) |
1031 | newud.ignore_checksums = True | ||
940 | newud.setup_localpath(ld) | 1032 | newud.setup_localpath(ld) |
941 | except bb.fetch2.BBFetchException as e: | 1033 | except bb.fetch2.BBFetchException as e: |
942 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) | 1034 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) |
@@ -1000,6 +1092,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1000 | # If that tarball is a local file:// we need to provide a symlink to it | 1092 | # If that tarball is a local file:// we need to provide a symlink to it |
1001 | dldir = ld.getVar("DL_DIR") | 1093 | dldir = ld.getVar("DL_DIR") |
1002 | 1094 | ||
1095 | if bb.utils.to_boolean(ld.getVar("BB_FETCH_PREMIRRORONLY")): | ||
1096 | ld = ld.createCopy() | ||
1097 | ld.setVar("BB_NO_NETWORK", "1") | ||
1098 | |||
1003 | if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): | 1099 | if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): |
1004 | # Create donestamp in old format to avoid triggering a re-download | 1100 | # Create donestamp in old format to avoid triggering a re-download |
1005 | if ud.donestamp: | 1101 | if ud.donestamp: |
@@ -1021,7 +1117,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1021 | origud.method.build_mirror_data(origud, ld) | 1117 | origud.method.build_mirror_data(origud, ld) |
1022 | return origud.localpath | 1118 | return origud.localpath |
1023 | # Otherwise the result is a local file:// and we symlink to it | 1119 | # Otherwise the result is a local file:// and we symlink to it |
1024 | ensure_symlink(ud.localpath, origud.localpath) | 1120 | # This may also be a link to a shallow archive |
1121 | # When using shallow mode, add a symlink to the original fullshallow | ||
1122 | # path to ensure a valid symlink even in the `PREMIRRORS` case | ||
1123 | origud.method.update_mirror_links(ud, origud) | ||
1025 | update_stamp(origud, ld) | 1124 | update_stamp(origud, ld) |
1026 | return ud.localpath | 1125 | return ud.localpath |
1027 | 1126 | ||
@@ -1046,7 +1145,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1046 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) | 1145 | logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) |
1047 | logger.debug(str(e)) | 1146 | logger.debug(str(e)) |
1048 | try: | 1147 | try: |
1049 | ud.method.clean(ud, ld) | 1148 | if ud.method.cleanup_upon_failure(): |
1149 | ud.method.clean(ud, ld) | ||
1050 | except UnboundLocalError: | 1150 | except UnboundLocalError: |
1051 | pass | 1151 | pass |
1052 | return False | 1152 | return False |
@@ -1054,23 +1154,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): | |||
1054 | if ud.lockfile and ud.lockfile != origud.lockfile: | 1154 | if ud.lockfile and ud.lockfile != origud.lockfile: |
1055 | bb.utils.unlockfile(lf) | 1155 | bb.utils.unlockfile(lf) |
1056 | 1156 | ||
1057 | |||
1058 | def ensure_symlink(target, link_name): | ||
1059 | if not os.path.exists(link_name): | ||
1060 | if os.path.islink(link_name): | ||
1061 | # Broken symbolic link | ||
1062 | os.unlink(link_name) | ||
1063 | |||
1064 | # In case this is executing without any file locks held (as is | ||
1065 | # the case for file:// URLs), two tasks may end up here at the | ||
1066 | # same time, in which case we do not want the second task to | ||
1067 | # fail when the link has already been created by the first task. | ||
1068 | try: | ||
1069 | os.symlink(target, link_name) | ||
1070 | except FileExistsError: | ||
1071 | pass | ||
1072 | |||
1073 | |||
1074 | def try_mirrors(fetch, d, origud, mirrors, check = False): | 1157 | def try_mirrors(fetch, d, origud, mirrors, check = False): |
1075 | """ | 1158 | """ |
1076 | Try to use a mirrored version of the sources. | 1159 | Try to use a mirrored version of the sources. |
@@ -1099,7 +1182,7 @@ def trusted_network(d, url): | |||
1099 | if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): | 1182 | if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): |
1100 | return True | 1183 | return True |
1101 | 1184 | ||
1102 | pkgname = d.expand(d.getVar('PN', False)) | 1185 | pkgname = d.getVar('PN') |
1103 | trusted_hosts = None | 1186 | trusted_hosts = None |
1104 | if pkgname: | 1187 | if pkgname: |
1105 | trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) | 1188 | trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) |
@@ -1140,11 +1223,11 @@ def srcrev_internal_helper(ud, d, name): | |||
1140 | pn = d.getVar("PN") | 1223 | pn = d.getVar("PN") |
1141 | attempts = [] | 1224 | attempts = [] |
1142 | if name != '' and pn: | 1225 | if name != '' and pn: |
1143 | attempts.append("SRCREV_%s_pn-%s" % (name, pn)) | 1226 | attempts.append("SRCREV_%s:pn-%s" % (name, pn)) |
1144 | if name != '': | 1227 | if name != '': |
1145 | attempts.append("SRCREV_%s" % name) | 1228 | attempts.append("SRCREV_%s" % name) |
1146 | if pn: | 1229 | if pn: |
1147 | attempts.append("SRCREV_pn-%s" % pn) | 1230 | attempts.append("SRCREV:pn-%s" % pn) |
1148 | attempts.append("SRCREV") | 1231 | attempts.append("SRCREV") |
1149 | 1232 | ||
1150 | for a in attempts: | 1233 | for a in attempts: |
@@ -1152,23 +1235,21 @@ def srcrev_internal_helper(ud, d, name): | |||
1152 | if srcrev and srcrev != "INVALID": | 1235 | if srcrev and srcrev != "INVALID": |
1153 | break | 1236 | break |
1154 | 1237 | ||
1155 | if 'rev' in ud.parm and 'tag' in ud.parm: | 1238 | if 'rev' in ud.parm: |
1156 | raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) | 1239 | parmrev = ud.parm['rev'] |
1157 | |||
1158 | if 'rev' in ud.parm or 'tag' in ud.parm: | ||
1159 | if 'rev' in ud.parm: | ||
1160 | parmrev = ud.parm['rev'] | ||
1161 | else: | ||
1162 | parmrev = ud.parm['tag'] | ||
1163 | if srcrev == "INVALID" or not srcrev: | 1240 | if srcrev == "INVALID" or not srcrev: |
1164 | return parmrev | 1241 | return parmrev |
1165 | if srcrev != parmrev: | 1242 | if srcrev != parmrev: |
1166 | raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) | 1243 | raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) |
1167 | return parmrev | 1244 | return parmrev |
1168 | 1245 | ||
1246 | if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev): | ||
1247 | return ud.parm['tag'] | ||
1248 | |||
1169 | if srcrev == "INVALID" or not srcrev: | 1249 | if srcrev == "INVALID" or not srcrev: |
1170 | raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) | 1250 | raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) |
1171 | if srcrev == "AUTOINC": | 1251 | if srcrev == "AUTOINC": |
1252 | d.setVar("__BBAUTOREV_ACTED_UPON", True) | ||
1172 | srcrev = ud.method.latest_revision(ud, d, name) | 1253 | srcrev = ud.method.latest_revision(ud, d, name) |
1173 | 1254 | ||
1174 | return srcrev | 1255 | return srcrev |
@@ -1180,23 +1261,21 @@ def get_checksum_file_list(d): | |||
1180 | SRC_URI as a space-separated string | 1261 | SRC_URI as a space-separated string |
1181 | """ | 1262 | """ |
1182 | fetch = Fetch([], d, cache = False, localonly = True) | 1263 | fetch = Fetch([], d, cache = False, localonly = True) |
1183 | |||
1184 | dl_dir = d.getVar('DL_DIR') | ||
1185 | filelist = [] | 1264 | filelist = [] |
1186 | for u in fetch.urls: | 1265 | for u in fetch.urls: |
1187 | ud = fetch.ud[u] | 1266 | ud = fetch.ud[u] |
1188 | |||
1189 | if ud and isinstance(ud.method, local.Local): | 1267 | if ud and isinstance(ud.method, local.Local): |
1190 | paths = ud.method.localpaths(ud, d) | 1268 | found = False |
1269 | paths = ud.method.localfile_searchpaths(ud, d) | ||
1191 | for f in paths: | 1270 | for f in paths: |
1192 | pth = ud.decodedurl | 1271 | pth = ud.path |
1193 | if f.startswith(dl_dir): | 1272 | if os.path.exists(f): |
1194 | # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else | 1273 | found = True |
1195 | if os.path.exists(f): | ||
1196 | bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) | ||
1197 | else: | ||
1198 | bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f))) | ||
1199 | filelist.append(f + ":" + str(os.path.exists(f))) | 1274 | filelist.append(f + ":" + str(os.path.exists(f))) |
1275 | if not found: | ||
1276 | bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found" | ||
1277 | "\nThe following paths were searched:" | ||
1278 | "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths))) | ||
1200 | 1279 | ||
1201 | return " ".join(filelist) | 1280 | return " ".join(filelist) |
1202 | 1281 | ||
@@ -1234,28 +1313,28 @@ class FetchData(object): | |||
1234 | self.setup = False | 1313 | self.setup = False |
1235 | 1314 | ||
1236 | def configure_checksum(checksum_id): | 1315 | def configure_checksum(checksum_id): |
1316 | checksum_plain_name = "%ssum" % checksum_id | ||
1237 | if "name" in self.parm: | 1317 | if "name" in self.parm: |
1238 | checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) | 1318 | checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) |
1239 | else: | 1319 | else: |
1240 | checksum_name = "%ssum" % checksum_id | 1320 | checksum_name = checksum_plain_name |
1241 | |||
1242 | setattr(self, "%s_name" % checksum_id, checksum_name) | ||
1243 | 1321 | ||
1244 | if checksum_name in self.parm: | 1322 | if checksum_name in self.parm: |
1245 | checksum_expected = self.parm[checksum_name] | 1323 | checksum_expected = self.parm[checksum_name] |
1246 | elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: | 1324 | elif checksum_plain_name in self.parm: |
1325 | checksum_expected = self.parm[checksum_plain_name] | ||
1326 | checksum_name = checksum_plain_name | ||
1327 | elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]: | ||
1247 | checksum_expected = None | 1328 | checksum_expected = None |
1248 | else: | 1329 | else: |
1249 | checksum_expected = d.getVarFlag("SRC_URI", checksum_name) | 1330 | checksum_expected = d.getVarFlag("SRC_URI", checksum_name) |
1250 | 1331 | ||
1332 | setattr(self, "%s_name" % checksum_id, checksum_name) | ||
1251 | setattr(self, "%s_expected" % checksum_id, checksum_expected) | 1333 | setattr(self, "%s_expected" % checksum_id, checksum_expected) |
1252 | 1334 | ||
1253 | for checksum_id in CHECKSUM_LIST: | 1335 | self.name = self.parm.get("name",'default') |
1254 | configure_checksum(checksum_id) | 1336 | if "," in self.name: |
1255 | 1337 | raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url) | |
1256 | self.ignore_checksums = False | ||
1257 | |||
1258 | self.names = self.parm.get("name",'default').split(',') | ||
1259 | 1338 | ||
1260 | self.method = None | 1339 | self.method = None |
1261 | for m in methods: | 1340 | for m in methods: |
@@ -1276,6 +1355,11 @@ class FetchData(object): | |||
1276 | if hasattr(self.method, "urldata_init"): | 1355 | if hasattr(self.method, "urldata_init"): |
1277 | self.method.urldata_init(self, d) | 1356 | self.method.urldata_init(self, d) |
1278 | 1357 | ||
1358 | for checksum_id in CHECKSUM_LIST: | ||
1359 | configure_checksum(checksum_id) | ||
1360 | |||
1361 | self.ignore_checksums = False | ||
1362 | |||
1279 | if "localpath" in self.parm: | 1363 | if "localpath" in self.parm: |
1280 | # if user sets localpath for file, use it instead. | 1364 | # if user sets localpath for file, use it instead. |
1281 | self.localpath = self.parm["localpath"] | 1365 | self.localpath = self.parm["localpath"] |
@@ -1302,13 +1386,7 @@ class FetchData(object): | |||
1302 | self.lockfile = basepath + '.lock' | 1386 | self.lockfile = basepath + '.lock' |
1303 | 1387 | ||
1304 | def setup_revisions(self, d): | 1388 | def setup_revisions(self, d): |
1305 | self.revisions = {} | 1389 | self.revision = srcrev_internal_helper(self, d, self.name) |
1306 | for name in self.names: | ||
1307 | self.revisions[name] = srcrev_internal_helper(self, d, name) | ||
1308 | |||
1309 | # add compatibility code for non name specified case | ||
1310 | if len(self.names) == 1: | ||
1311 | self.revision = self.revisions[self.names[0]] | ||
1312 | 1390 | ||
1313 | def setup_localpath(self, d): | 1391 | def setup_localpath(self, d): |
1314 | if not self.localpath: | 1392 | if not self.localpath: |
@@ -1355,6 +1433,9 @@ class FetchMethod(object): | |||
1355 | Is localpath something that can be represented by a checksum? | 1433 | Is localpath something that can be represented by a checksum? |
1356 | """ | 1434 | """ |
1357 | 1435 | ||
1436 | # We cannot compute checksums for None | ||
1437 | if urldata.localpath is None: | ||
1438 | return False | ||
1358 | # We cannot compute checksums for directories | 1439 | # We cannot compute checksums for directories |
1359 | if os.path.isdir(urldata.localpath): | 1440 | if os.path.isdir(urldata.localpath): |
1360 | return False | 1441 | return False |
@@ -1367,6 +1448,12 @@ class FetchMethod(object): | |||
1367 | """ | 1448 | """ |
1368 | return False | 1449 | return False |
1369 | 1450 | ||
1451 | def cleanup_upon_failure(self): | ||
1452 | """ | ||
1453 | When a fetch fails, should clean() be called? | ||
1454 | """ | ||
1455 | return True | ||
1456 | |||
1370 | def verify_donestamp(self, ud, d): | 1457 | def verify_donestamp(self, ud, d): |
1371 | """ | 1458 | """ |
1372 | Verify the donestamp file | 1459 | Verify the donestamp file |
@@ -1427,37 +1514,40 @@ class FetchMethod(object): | |||
1427 | (file, urldata.parm.get('unpack'))) | 1514 | (file, urldata.parm.get('unpack'))) |
1428 | 1515 | ||
1429 | base, ext = os.path.splitext(file) | 1516 | base, ext = os.path.splitext(file) |
1430 | if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']: | 1517 | if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']: |
1431 | efile = os.path.join(rootdir, os.path.basename(base)) | 1518 | efile = os.path.join(rootdir, os.path.basename(base)) |
1432 | else: | 1519 | else: |
1433 | efile = file | 1520 | efile = file |
1434 | cmd = None | 1521 | cmd = None |
1435 | 1522 | ||
1436 | if unpack: | 1523 | if unpack: |
1524 | tar_cmd = 'tar --extract --no-same-owner' | ||
1525 | if 'striplevel' in urldata.parm: | ||
1526 | tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel'] | ||
1437 | if file.endswith('.tar'): | 1527 | if file.endswith('.tar'): |
1438 | cmd = 'tar x --no-same-owner -f %s' % file | 1528 | cmd = '%s -f %s' % (tar_cmd, file) |
1439 | elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): | 1529 | elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): |
1440 | cmd = 'tar xz --no-same-owner -f %s' % file | 1530 | cmd = '%s -z -f %s' % (tar_cmd, file) |
1441 | elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): | 1531 | elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): |
1442 | cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file | 1532 | cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd) |
1443 | elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): | 1533 | elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): |
1444 | cmd = 'gzip -dc %s > %s' % (file, efile) | 1534 | cmd = 'gzip -dc %s > %s' % (file, efile) |
1445 | elif file.endswith('.bz2'): | 1535 | elif file.endswith('.bz2'): |
1446 | cmd = 'bzip2 -dc %s > %s' % (file, efile) | 1536 | cmd = 'bzip2 -dc %s > %s' % (file, efile) |
1447 | elif file.endswith('.txz') or file.endswith('.tar.xz'): | 1537 | elif file.endswith('.txz') or file.endswith('.tar.xz'): |
1448 | cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file | 1538 | cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd) |
1449 | elif file.endswith('.xz'): | 1539 | elif file.endswith('.xz'): |
1450 | cmd = 'xz -dc %s > %s' % (file, efile) | 1540 | cmd = 'xz -dc %s > %s' % (file, efile) |
1451 | elif file.endswith('.tar.lz'): | 1541 | elif file.endswith('.tar.lz'): |
1452 | cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file | 1542 | cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd) |
1453 | elif file.endswith('.lz'): | 1543 | elif file.endswith('.lz'): |
1454 | cmd = 'lzip -dc %s > %s' % (file, efile) | 1544 | cmd = 'lzip -dc %s > %s' % (file, efile) |
1455 | elif file.endswith('.tar.7z'): | 1545 | elif file.endswith('.tar.7z'): |
1456 | cmd = '7z x -so %s | tar x --no-same-owner -f -' % file | 1546 | cmd = '7z x -so %s | %s -f -' % (file, tar_cmd) |
1457 | elif file.endswith('.7z'): | 1547 | elif file.endswith('.7z'): |
1458 | cmd = '7za x -y %s 1>/dev/null' % file | 1548 | cmd = '7za x -y %s 1>/dev/null' % file |
1459 | elif file.endswith('.tzst') or file.endswith('.tar.zst'): | 1549 | elif file.endswith('.tzst') or file.endswith('.tar.zst'): |
1460 | cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file | 1550 | cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd) |
1461 | elif file.endswith('.zst'): | 1551 | elif file.endswith('.zst'): |
1462 | cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) | 1552 | cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) |
1463 | elif file.endswith('.zip') or file.endswith('.jar'): | 1553 | elif file.endswith('.zip') or file.endswith('.jar'): |
@@ -1483,14 +1573,14 @@ class FetchMethod(object): | |||
1483 | datafile = None | 1573 | datafile = None |
1484 | if output: | 1574 | if output: |
1485 | for line in output.decode().splitlines(): | 1575 | for line in output.decode().splitlines(): |
1486 | if line.startswith('data.tar.'): | 1576 | if line.startswith('data.tar.') or line == 'data.tar': |
1487 | datafile = line | 1577 | datafile = line |
1488 | break | 1578 | break |
1489 | else: | 1579 | else: |
1490 | raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) | 1580 | raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar* file", urldata.url) |
1491 | else: | 1581 | else: |
1492 | raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) | 1582 | raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) |
1493 | cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile) | 1583 | cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile) |
1494 | 1584 | ||
1495 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | 1585 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd |
1496 | if 'subdir' in urldata.parm: | 1586 | if 'subdir' in urldata.parm: |
@@ -1506,6 +1596,7 @@ class FetchMethod(object): | |||
1506 | unpackdir = rootdir | 1596 | unpackdir = rootdir |
1507 | 1597 | ||
1508 | if not unpack or not cmd: | 1598 | if not unpack or not cmd: |
1599 | urldata.unpack_tracer.unpack("file-copy", unpackdir) | ||
1509 | # If file == dest, then avoid any copies, as we already put the file into dest! | 1600 | # If file == dest, then avoid any copies, as we already put the file into dest! |
1510 | dest = os.path.join(unpackdir, os.path.basename(file)) | 1601 | dest = os.path.join(unpackdir, os.path.basename(file)) |
1511 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): | 1602 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): |
@@ -1519,7 +1610,9 @@ class FetchMethod(object): | |||
1519 | if urlpath.find("/") != -1: | 1610 | if urlpath.find("/") != -1: |
1520 | destdir = urlpath.rsplit("/", 1)[0] + '/' | 1611 | destdir = urlpath.rsplit("/", 1)[0] + '/' |
1521 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) | 1612 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) |
1522 | cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) | 1613 | cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir) |
1614 | else: | ||
1615 | urldata.unpack_tracer.unpack("archive-extract", unpackdir) | ||
1523 | 1616 | ||
1524 | if not cmd: | 1617 | if not cmd: |
1525 | return | 1618 | return |
@@ -1546,6 +1639,28 @@ class FetchMethod(object): | |||
1546 | """ | 1639 | """ |
1547 | bb.utils.remove(urldata.localpath) | 1640 | bb.utils.remove(urldata.localpath) |
1548 | 1641 | ||
1642 | def ensure_symlink(self, target, link_name): | ||
1643 | if not os.path.exists(link_name): | ||
1644 | dirname = os.path.dirname(link_name) | ||
1645 | bb.utils.mkdirhier(dirname) | ||
1646 | if os.path.islink(link_name): | ||
1647 | # Broken symbolic link | ||
1648 | os.unlink(link_name) | ||
1649 | |||
1650 | # In case this is executing without any file locks held (as is | ||
1651 | # the case for file:// URLs), two tasks may end up here at the | ||
1652 | # same time, in which case we do not want the second task to | ||
1653 | # fail when the link has already been created by the first task. | ||
1654 | try: | ||
1655 | os.symlink(target, link_name) | ||
1656 | except FileExistsError: | ||
1657 | pass | ||
1658 | |||
1659 | def update_mirror_links(self, ud, origud): | ||
1660 | # For local file:// results, create a symlink to them | ||
1661 | # This may also be a link to a shallow archive | ||
1662 | self.ensure_symlink(ud.localpath, origud.localpath) | ||
1663 | |||
1549 | def try_premirror(self, urldata, d): | 1664 | def try_premirror(self, urldata, d): |
1550 | """ | 1665 | """ |
1551 | Should premirrors be used? | 1666 | Should premirrors be used? |
@@ -1573,13 +1688,13 @@ class FetchMethod(object): | |||
1573 | if not hasattr(self, "_latest_revision"): | 1688 | if not hasattr(self, "_latest_revision"): |
1574 | raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) | 1689 | raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) |
1575 | 1690 | ||
1576 | revs = bb.persist_data.persist('BB_URI_HEADREVS', d) | ||
1577 | key = self.generate_revision_key(ud, d, name) | 1691 | key = self.generate_revision_key(ud, d, name) |
1578 | try: | 1692 | |
1579 | return revs[key] | 1693 | rev = _revisions_cache.get_rev(key) |
1580 | except KeyError: | 1694 | if rev is None: |
1581 | revs[key] = rev = self._latest_revision(ud, d, name) | 1695 | rev = self._latest_revision(ud, d, name) |
1582 | return rev | 1696 | _revisions_cache.set_rev(key, rev) |
1697 | return rev | ||
1583 | 1698 | ||
1584 | def sortable_revision(self, ud, d, name): | 1699 | def sortable_revision(self, ud, d, name): |
1585 | latest_rev = self._build_revision(ud, d, name) | 1700 | latest_rev = self._build_revision(ud, d, name) |
@@ -1611,12 +1726,61 @@ class FetchMethod(object): | |||
1611 | """ | 1726 | """ |
1612 | return [] | 1727 | return [] |
1613 | 1728 | ||
1729 | |||
1730 | class DummyUnpackTracer(object): | ||
1731 | """ | ||
1732 | Abstract API definition for a class that traces unpacked source files back | ||
1733 | to their respective upstream SRC_URI entries, for software composition | ||
1734 | analysis, license compliance and detailed SBOM generation purposes. | ||
1735 | User may load their own unpack tracer class (instead of the dummy | ||
1736 | one) by setting the BB_UNPACK_TRACER_CLASS config parameter. | ||
1737 | """ | ||
1738 | def start(self, unpackdir, urldata_dict, d): | ||
1739 | """ | ||
1740 | Start tracing the core Fetch.unpack process, using an index to map | ||
1741 | unpacked files to each SRC_URI entry. | ||
1742 | This method is called by Fetch.unpack and it may receive nested calls by | ||
1743 | gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit | ||
1744 | URLs and by recursively calling Fetch.unpack from new (nested) Fetch | ||
1745 | instances. | ||
1746 | """ | ||
1747 | return | ||
1748 | def start_url(self, url): | ||
1749 | """Start tracing url unpack process. | ||
1750 | This method is called by Fetch.unpack before the fetcher-specific unpack | ||
1751 | method starts, and it may receive nested calls by gitsm and npmsw | ||
1752 | fetchers. | ||
1753 | """ | ||
1754 | return | ||
1755 | def unpack(self, unpack_type, destdir): | ||
1756 | """ | ||
1757 | Set unpack_type and destdir for current url. | ||
1758 | This method is called by the fetcher-specific unpack method after url | ||
1759 | tracing started. | ||
1760 | """ | ||
1761 | return | ||
1762 | def finish_url(self, url): | ||
1763 | """Finish tracing url unpack process and update the file index. | ||
1764 | This method is called by Fetch.unpack after the fetcher-specific unpack | ||
1765 | method finished its job, and it may receive nested calls by gitsm | ||
1766 | and npmsw fetchers. | ||
1767 | """ | ||
1768 | return | ||
1769 | def complete(self): | ||
1770 | """ | ||
1771 | Finish tracing the Fetch.unpack process, and check if all nested | ||
1772 | Fecth.unpack calls (if any) have been completed; if so, save collected | ||
1773 | metadata. | ||
1774 | """ | ||
1775 | return | ||
1776 | |||
1777 | |||
1614 | class Fetch(object): | 1778 | class Fetch(object): |
1615 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): | 1779 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): |
1616 | if localonly and cache: | 1780 | if localonly and cache: |
1617 | raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") | 1781 | raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") |
1618 | 1782 | ||
1619 | if len(urls) == 0: | 1783 | if not urls: |
1620 | urls = d.getVar("SRC_URI").split() | 1784 | urls = d.getVar("SRC_URI").split() |
1621 | self.urls = urls | 1785 | self.urls = urls |
1622 | self.d = d | 1786 | self.d = d |
@@ -1631,10 +1795,30 @@ class Fetch(object): | |||
1631 | if key in urldata_cache: | 1795 | if key in urldata_cache: |
1632 | self.ud = urldata_cache[key] | 1796 | self.ud = urldata_cache[key] |
1633 | 1797 | ||
1798 | # the unpack_tracer object needs to be made available to possible nested | ||
1799 | # Fetch instances (when those are created by gitsm and npmsw fetchers) | ||
1800 | # so we set it as a global variable | ||
1801 | global unpack_tracer | ||
1802 | try: | ||
1803 | unpack_tracer | ||
1804 | except NameError: | ||
1805 | class_path = d.getVar("BB_UNPACK_TRACER_CLASS") | ||
1806 | if class_path: | ||
1807 | # use user-defined unpack tracer class | ||
1808 | import importlib | ||
1809 | module_name, _, class_name = class_path.rpartition(".") | ||
1810 | module = importlib.import_module(module_name) | ||
1811 | class_ = getattr(module, class_name) | ||
1812 | unpack_tracer = class_() | ||
1813 | else: | ||
1814 | # fall back to the dummy/abstract class | ||
1815 | unpack_tracer = DummyUnpackTracer() | ||
1816 | |||
1634 | for url in urls: | 1817 | for url in urls: |
1635 | if url not in self.ud: | 1818 | if url not in self.ud: |
1636 | try: | 1819 | try: |
1637 | self.ud[url] = FetchData(url, d, localonly) | 1820 | self.ud[url] = FetchData(url, d, localonly) |
1821 | self.ud[url].unpack_tracer = unpack_tracer | ||
1638 | except NonLocalMethod: | 1822 | except NonLocalMethod: |
1639 | if localonly: | 1823 | if localonly: |
1640 | self.ud[url] = None | 1824 | self.ud[url] = None |
@@ -1648,7 +1832,7 @@ class Fetch(object): | |||
1648 | self.ud[url] = FetchData(url, self.d) | 1832 | self.ud[url] = FetchData(url, self.d) |
1649 | 1833 | ||
1650 | self.ud[url].setup_localpath(self.d) | 1834 | self.ud[url].setup_localpath(self.d) |
1651 | return self.d.expand(self.ud[url].localpath) | 1835 | return self.ud[url].localpath |
1652 | 1836 | ||
1653 | def localpaths(self): | 1837 | def localpaths(self): |
1654 | """ | 1838 | """ |
@@ -1673,6 +1857,7 @@ class Fetch(object): | |||
1673 | network = self.d.getVar("BB_NO_NETWORK") | 1857 | network = self.d.getVar("BB_NO_NETWORK") |
1674 | premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) | 1858 | premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) |
1675 | 1859 | ||
1860 | checksum_missing_messages = [] | ||
1676 | for u in urls: | 1861 | for u in urls: |
1677 | ud = self.ud[u] | 1862 | ud = self.ud[u] |
1678 | ud.setup_localpath(self.d) | 1863 | ud.setup_localpath(self.d) |
@@ -1684,7 +1869,6 @@ class Fetch(object): | |||
1684 | 1869 | ||
1685 | try: | 1870 | try: |
1686 | self.d.setVar("BB_NO_NETWORK", network) | 1871 | self.d.setVar("BB_NO_NETWORK", network) |
1687 | |||
1688 | if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): | 1872 | if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): |
1689 | done = True | 1873 | done = True |
1690 | elif m.try_premirror(ud, self.d): | 1874 | elif m.try_premirror(ud, self.d): |
@@ -1701,23 +1885,28 @@ class Fetch(object): | |||
1701 | logger.debug(str(e)) | 1885 | logger.debug(str(e)) |
1702 | done = False | 1886 | done = False |
1703 | 1887 | ||
1888 | d = self.d | ||
1704 | if premirroronly: | 1889 | if premirroronly: |
1705 | self.d.setVar("BB_NO_NETWORK", "1") | 1890 | # Only disable the network in a copy |
1891 | d = bb.data.createCopy(self.d) | ||
1892 | d.setVar("BB_NO_NETWORK", "1") | ||
1706 | 1893 | ||
1707 | firsterr = None | 1894 | firsterr = None |
1708 | verified_stamp = m.verify_donestamp(ud, self.d) | 1895 | verified_stamp = False |
1709 | if not done and (not verified_stamp or m.need_update(ud, self.d)): | 1896 | if done: |
1897 | verified_stamp = m.verify_donestamp(ud, d) | ||
1898 | if not done and (not verified_stamp or m.need_update(ud, d)): | ||
1710 | try: | 1899 | try: |
1711 | if not trusted_network(self.d, ud.url): | 1900 | if not trusted_network(d, ud.url): |
1712 | raise UntrustedUrl(ud.url) | 1901 | raise UntrustedUrl(ud.url) |
1713 | logger.debug("Trying Upstream") | 1902 | logger.debug("Trying Upstream") |
1714 | m.download(ud, self.d) | 1903 | m.download(ud, d) |
1715 | if hasattr(m, "build_mirror_data"): | 1904 | if hasattr(m, "build_mirror_data"): |
1716 | m.build_mirror_data(ud, self.d) | 1905 | m.build_mirror_data(ud, d) |
1717 | done = True | 1906 | done = True |
1718 | # early checksum verify, so that if checksum mismatched, | 1907 | # early checksum verify, so that if checksum mismatched, |
1719 | # fetcher still have chance to fetch from mirror | 1908 | # fetcher still have chance to fetch from mirror |
1720 | m.update_donestamp(ud, self.d) | 1909 | m.update_donestamp(ud, d) |
1721 | 1910 | ||
1722 | except bb.fetch2.NetworkAccess: | 1911 | except bb.fetch2.NetworkAccess: |
1723 | raise | 1912 | raise |
@@ -1735,18 +1924,18 @@ class Fetch(object): | |||
1735 | logger.debug(str(e)) | 1924 | logger.debug(str(e)) |
1736 | firsterr = e | 1925 | firsterr = e |
1737 | # Remove any incomplete fetch | 1926 | # Remove any incomplete fetch |
1738 | if not verified_stamp: | 1927 | if not verified_stamp and m.cleanup_upon_failure(): |
1739 | m.clean(ud, self.d) | 1928 | m.clean(ud, d) |
1740 | logger.debug("Trying MIRRORS") | 1929 | logger.debug("Trying MIRRORS") |
1741 | mirrors = mirror_from_string(self.d.getVar('MIRRORS')) | 1930 | mirrors = mirror_from_string(d.getVar('MIRRORS')) |
1742 | done = m.try_mirrors(self, ud, self.d, mirrors) | 1931 | done = m.try_mirrors(self, ud, d, mirrors) |
1743 | 1932 | ||
1744 | if not done or not m.done(ud, self.d): | 1933 | if not done or not m.done(ud, d): |
1745 | if firsterr: | 1934 | if firsterr: |
1746 | logger.error(str(firsterr)) | 1935 | logger.error(str(firsterr)) |
1747 | raise FetchError("Unable to fetch URL from any source.", u) | 1936 | raise FetchError("Unable to fetch URL from any source.", u) |
1748 | 1937 | ||
1749 | m.update_donestamp(ud, self.d) | 1938 | m.update_donestamp(ud, d) |
1750 | 1939 | ||
1751 | except IOError as e: | 1940 | except IOError as e: |
1752 | if e.errno in [errno.ESTALE]: | 1941 | if e.errno in [errno.ESTALE]: |
@@ -1754,17 +1943,28 @@ class Fetch(object): | |||
1754 | raise ChecksumError("Stale Error Detected") | 1943 | raise ChecksumError("Stale Error Detected") |
1755 | 1944 | ||
1756 | except BBFetchException as e: | 1945 | except BBFetchException as e: |
1757 | if isinstance(e, ChecksumError): | 1946 | if isinstance(e, NoChecksumError): |
1947 | (message, _) = e.args | ||
1948 | checksum_missing_messages.append(message) | ||
1949 | continue | ||
1950 | elif isinstance(e, ChecksumError): | ||
1758 | logger.error("Checksum failure fetching %s" % u) | 1951 | logger.error("Checksum failure fetching %s" % u) |
1759 | raise | 1952 | raise |
1760 | 1953 | ||
1761 | finally: | 1954 | finally: |
1762 | if ud.lockfile: | 1955 | if ud.lockfile: |
1763 | bb.utils.unlockfile(lf) | 1956 | bb.utils.unlockfile(lf) |
1957 | if checksum_missing_messages: | ||
1958 | logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages)) | ||
1959 | raise BBFetchException("There was some missing checksums in the recipe") | ||
1764 | 1960 | ||
1765 | def checkstatus(self, urls=None): | 1961 | def checkstatus(self, urls=None): |
1766 | """ | 1962 | """ |
1767 | Check all urls exist upstream | 1963 | Check all URLs exist upstream. |
1964 | |||
1965 | Returns None if the URLs exist, raises FetchError if the check wasn't | ||
1966 | successful but there wasn't an error (such as file not found), and | ||
1967 | raises other exceptions in error cases. | ||
1768 | """ | 1968 | """ |
1769 | 1969 | ||
1770 | if not urls: | 1970 | if not urls: |
@@ -1787,7 +1987,7 @@ class Fetch(object): | |||
1787 | ret = m.try_mirrors(self, ud, self.d, mirrors, True) | 1987 | ret = m.try_mirrors(self, ud, self.d, mirrors, True) |
1788 | 1988 | ||
1789 | if not ret: | 1989 | if not ret: |
1790 | raise FetchError("URL %s doesn't work" % u, u) | 1990 | raise FetchError("URL doesn't work", u) |
1791 | 1991 | ||
1792 | def unpack(self, root, urls=None): | 1992 | def unpack(self, root, urls=None): |
1793 | """ | 1993 | """ |
@@ -1797,6 +1997,8 @@ class Fetch(object): | |||
1797 | if not urls: | 1997 | if not urls: |
1798 | urls = self.urls | 1998 | urls = self.urls |
1799 | 1999 | ||
2000 | unpack_tracer.start(root, self.ud, self.d) | ||
2001 | |||
1800 | for u in urls: | 2002 | for u in urls: |
1801 | ud = self.ud[u] | 2003 | ud = self.ud[u] |
1802 | ud.setup_localpath(self.d) | 2004 | ud.setup_localpath(self.d) |
@@ -1804,11 +2006,15 @@ class Fetch(object): | |||
1804 | if ud.lockfile: | 2006 | if ud.lockfile: |
1805 | lf = bb.utils.lockfile(ud.lockfile) | 2007 | lf = bb.utils.lockfile(ud.lockfile) |
1806 | 2008 | ||
2009 | unpack_tracer.start_url(u) | ||
1807 | ud.method.unpack(ud, root, self.d) | 2010 | ud.method.unpack(ud, root, self.d) |
2011 | unpack_tracer.finish_url(u) | ||
1808 | 2012 | ||
1809 | if ud.lockfile: | 2013 | if ud.lockfile: |
1810 | bb.utils.unlockfile(lf) | 2014 | bb.utils.unlockfile(lf) |
1811 | 2015 | ||
2016 | unpack_tracer.complete() | ||
2017 | |||
1812 | def clean(self, urls=None): | 2018 | def clean(self, urls=None): |
1813 | """ | 2019 | """ |
1814 | Clean files that the fetcher gets or places | 2020 | Clean files that the fetcher gets or places |
@@ -1908,6 +2114,10 @@ from . import repo | |||
1908 | from . import clearcase | 2114 | from . import clearcase |
1909 | from . import npm | 2115 | from . import npm |
1910 | from . import npmsw | 2116 | from . import npmsw |
2117 | from . import az | ||
2118 | from . import crate | ||
2119 | from . import gcp | ||
2120 | from . import gomod | ||
1911 | 2121 | ||
1912 | methods.append(local.Local()) | 2122 | methods.append(local.Local()) |
1913 | methods.append(wget.Wget()) | 2123 | methods.append(wget.Wget()) |
@@ -1927,3 +2137,8 @@ methods.append(repo.Repo()) | |||
1927 | methods.append(clearcase.ClearCase()) | 2137 | methods.append(clearcase.ClearCase()) |
1928 | methods.append(npm.Npm()) | 2138 | methods.append(npm.Npm()) |
1929 | methods.append(npmsw.NpmShrinkWrap()) | 2139 | methods.append(npmsw.NpmShrinkWrap()) |
2140 | methods.append(az.Az()) | ||
2141 | methods.append(crate.Crate()) | ||
2142 | methods.append(gcp.GCP()) | ||
2143 | methods.append(gomod.GoMod()) | ||
2144 | methods.append(gomod.GoModGit()) | ||
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py new file mode 100644 index 0000000000..1d3664f213 --- /dev/null +++ b/bitbake/lib/bb/fetch2/az.py | |||
@@ -0,0 +1,98 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' Azure Storage implementation | ||
3 | |||
4 | """ | ||
5 | |||
6 | # Copyright (C) 2021 Alejandro Hernandez Samaniego | ||
7 | # | ||
8 | # Based on bb.fetch2.wget: | ||
9 | # Copyright (C) 2003, 2004 Chris Larson | ||
10 | # | ||
11 | # SPDX-License-Identifier: GPL-2.0-only | ||
12 | # | ||
13 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
14 | |||
15 | import shlex | ||
16 | import os | ||
17 | import bb | ||
18 | from bb.fetch2 import FetchError | ||
19 | from bb.fetch2 import logger | ||
20 | from bb.fetch2.wget import Wget | ||
21 | |||
22 | |||
23 | class Az(Wget): | ||
24 | |||
25 | def supports(self, ud, d): | ||
26 | """ | ||
27 | Check to see if a given url can be fetched from Azure Storage | ||
28 | """ | ||
29 | return ud.type in ['az'] | ||
30 | |||
31 | |||
32 | def checkstatus(self, fetch, ud, d, try_again=True): | ||
33 | |||
34 | # checkstatus discards parameters either way, we need to do this before adding the SAS | ||
35 | ud.url = ud.url.replace('az://','https://').split(';')[0] | ||
36 | |||
37 | az_sas = d.getVar('AZ_SAS') | ||
38 | if az_sas and az_sas not in ud.url: | ||
39 | if not az_sas.startswith('?'): | ||
40 | raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.") | ||
41 | ud.url += az_sas | ||
42 | |||
43 | return Wget.checkstatus(self, fetch, ud, d, try_again) | ||
44 | |||
45 | # Override download method, include retries | ||
46 | def download(self, ud, d, retries=3): | ||
47 | """Fetch urls""" | ||
48 | |||
49 | # If were reaching the account transaction limit we might be refused a connection, | ||
50 | # retrying allows us to avoid false negatives since the limit changes over time | ||
51 | fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' | ||
52 | |||
53 | # We need to provide a localpath to avoid wget using the SAS | ||
54 | # ud.localfile either has the downloadfilename or ud.path | ||
55 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | ||
56 | bb.utils.mkdirhier(os.path.dirname(localpath)) | ||
57 | fetchcmd += " -O %s" % shlex.quote(localpath) | ||
58 | |||
59 | |||
60 | if ud.user and ud.pswd: | ||
61 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | ||
62 | |||
63 | # Check if a Shared Access Signature was given and use it | ||
64 | az_sas = d.getVar('AZ_SAS') | ||
65 | |||
66 | if az_sas: | ||
67 | if not az_sas.startswith('?'): | ||
68 | raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.") | ||
69 | azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) | ||
70 | else: | ||
71 | azuri = '%s%s%s' % ('https://', ud.host, ud.path) | ||
72 | |||
73 | dldir = d.getVar("DL_DIR") | ||
74 | if os.path.exists(ud.localpath): | ||
75 | # file exists, but we didnt complete it.. trying again. | ||
76 | fetchcmd += " -c -P %s '%s'" % (dldir, azuri) | ||
77 | else: | ||
78 | fetchcmd += " -P %s '%s'" % (dldir, azuri) | ||
79 | |||
80 | try: | ||
81 | self._runwget(ud, d, fetchcmd, False) | ||
82 | except FetchError as e: | ||
83 | # Azure fails on handshake sometimes when using wget after some stress, producing a | ||
84 | # FetchError from the fetcher, if the artifact exists retyring should succeed | ||
85 | if 'Unable to establish SSL connection' in str(e): | ||
86 | logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) | ||
87 | self.download(ud, d, retries -1) | ||
88 | |||
89 | # Sanity check since wget can pretend it succeed when it didn't | ||
90 | # Also, this used to happen if sourceforge sent us to the mirror page | ||
91 | if not os.path.exists(ud.localpath): | ||
92 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) | ||
93 | |||
94 | if os.path.getsize(ud.localpath) == 0: | ||
95 | os.remove(ud.localpath) | ||
96 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) | ||
97 | |||
98 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/clearcase.py b/bitbake/lib/bb/fetch2/clearcase.py index 1a9c863769..17500daf95 100644 --- a/bitbake/lib/bb/fetch2/clearcase.py +++ b/bitbake/lib/bb/fetch2/clearcase.py | |||
@@ -108,7 +108,7 @@ class ClearCase(FetchMethod): | |||
108 | ud.module.replace("/", "."), | 108 | ud.module.replace("/", "."), |
109 | ud.label.replace("/", ".")) | 109 | ud.label.replace("/", ".")) |
110 | 110 | ||
111 | ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True)) | 111 | ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME")) |
112 | ud.csname = "%s-config-spec" % (ud.identifier) | 112 | ud.csname = "%s-config-spec" % (ud.identifier) |
113 | ud.ccasedir = os.path.join(d.getVar("DL_DIR"), ud.type) | 113 | ud.ccasedir = os.path.join(d.getVar("DL_DIR"), ud.type) |
114 | ud.viewdir = os.path.join(ud.ccasedir, ud.viewname) | 114 | ud.viewdir = os.path.join(ud.ccasedir, ud.viewname) |
@@ -130,8 +130,6 @@ class ClearCase(FetchMethod): | |||
130 | self.debug("configspecfile = %s" % ud.configspecfile) | 130 | self.debug("configspecfile = %s" % ud.configspecfile) |
131 | self.debug("localfile = %s" % ud.localfile) | 131 | self.debug("localfile = %s" % ud.localfile) |
132 | 132 | ||
133 | ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile) | ||
134 | |||
135 | def _build_ccase_command(self, ud, command): | 133 | def _build_ccase_command(self, ud, command): |
136 | """ | 134 | """ |
137 | Build up a commandline based on ud | 135 | Build up a commandline based on ud |
@@ -196,7 +194,7 @@ class ClearCase(FetchMethod): | |||
196 | 194 | ||
197 | def need_update(self, ud, d): | 195 | def need_update(self, ud, d): |
198 | if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec): | 196 | if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec): |
199 | ud.identifier += "-%s" % d.getVar("DATETIME",d, True) | 197 | ud.identifier += "-%s" % d.getVar("DATETIME") |
200 | return True | 198 | return True |
201 | if os.path.exists(ud.localpath): | 199 | if os.path.exists(ud.localpath): |
202 | return False | 200 | return False |
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py new file mode 100644 index 0000000000..e611736f06 --- /dev/null +++ b/bitbake/lib/bb/fetch2/crate.py | |||
@@ -0,0 +1,150 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake 'Fetch' implementation for crates.io | ||
5 | """ | ||
6 | |||
7 | # Copyright (C) 2016 Doug Goldstein | ||
8 | # | ||
9 | # SPDX-License-Identifier: GPL-2.0-only | ||
10 | # | ||
11 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
12 | |||
13 | import hashlib | ||
14 | import json | ||
15 | import os | ||
16 | import subprocess | ||
17 | import bb | ||
18 | from bb.fetch2 import logger, subprocess_setup, UnpackError | ||
19 | from bb.fetch2.wget import Wget | ||
20 | |||
21 | |||
22 | class Crate(Wget): | ||
23 | |||
24 | """Class to fetch crates via wget""" | ||
25 | |||
26 | def _cargo_bitbake_path(self, rootdir): | ||
27 | return os.path.join(rootdir, "cargo_home", "bitbake") | ||
28 | |||
29 | def supports(self, ud, d): | ||
30 | """ | ||
31 | Check to see if a given url is for this fetcher | ||
32 | """ | ||
33 | return ud.type in ['crate'] | ||
34 | |||
35 | def recommends_checksum(self, urldata): | ||
36 | return True | ||
37 | |||
38 | def urldata_init(self, ud, d): | ||
39 | """ | ||
40 | Sets up to download the respective crate from crates.io | ||
41 | """ | ||
42 | |||
43 | if ud.type == 'crate': | ||
44 | self._crate_urldata_init(ud, d) | ||
45 | |||
46 | super(Crate, self).urldata_init(ud, d) | ||
47 | |||
48 | def _crate_urldata_init(self, ud, d): | ||
49 | """ | ||
50 | Sets up the download for a crate | ||
51 | """ | ||
52 | |||
53 | # URL syntax is: crate://NAME/VERSION | ||
54 | # break the URL apart by / | ||
55 | parts = ud.url.split('/') | ||
56 | if len(parts) < 5: | ||
57 | raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url) | ||
58 | |||
59 | # version is expected to be the last token | ||
60 | # but ignore possible url parameters which will be used | ||
61 | # by the top fetcher class | ||
62 | version = parts[-1].split(";")[0] | ||
63 | # second to last field is name | ||
64 | name = parts[-2] | ||
65 | # host (this is to allow custom crate registries to be specified | ||
66 | host = '/'.join(parts[2:-2]) | ||
67 | |||
68 | # if using upstream just fix it up nicely | ||
69 | if host == 'crates.io': | ||
70 | host = 'crates.io/api/v1/crates' | ||
71 | |||
72 | ud.url = "https://%s/%s/%s/download" % (host, name, version) | ||
73 | ud.versionsurl = "https://%s/%s/versions" % (host, name) | ||
74 | ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) | ||
75 | if 'name' not in ud.parm: | ||
76 | ud.parm['name'] = '%s-%s' % (name, version) | ||
77 | |||
78 | logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename'])) | ||
79 | |||
80 | def unpack(self, ud, rootdir, d): | ||
81 | """ | ||
82 | Uses the crate to build the necessary paths for cargo to utilize it | ||
83 | """ | ||
84 | if ud.type == 'crate': | ||
85 | return self._crate_unpack(ud, rootdir, d) | ||
86 | else: | ||
87 | super(Crate, self).unpack(ud, rootdir, d) | ||
88 | |||
89 | def _crate_unpack(self, ud, rootdir, d): | ||
90 | """ | ||
91 | Unpacks a crate | ||
92 | """ | ||
93 | thefile = ud.localpath | ||
94 | |||
95 | # possible metadata we need to write out | ||
96 | metadata = {} | ||
97 | |||
98 | # change to the rootdir to unpack but save the old working dir | ||
99 | save_cwd = os.getcwd() | ||
100 | os.chdir(rootdir) | ||
101 | |||
102 | bp = d.getVar('BP') | ||
103 | if bp == ud.parm.get('name'): | ||
104 | cmd = "tar -xz --no-same-owner -f %s" % thefile | ||
105 | ud.unpack_tracer.unpack("crate-extract", rootdir) | ||
106 | else: | ||
107 | cargo_bitbake = self._cargo_bitbake_path(rootdir) | ||
108 | ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake) | ||
109 | |||
110 | cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) | ||
111 | |||
112 | # ensure we've got these paths made | ||
113 | bb.utils.mkdirhier(cargo_bitbake) | ||
114 | |||
115 | # generate metadata necessary | ||
116 | with open(thefile, 'rb') as f: | ||
117 | # get the SHA256 of the original tarball | ||
118 | tarhash = hashlib.sha256(f.read()).hexdigest() | ||
119 | |||
120 | metadata['files'] = {} | ||
121 | metadata['package'] = tarhash | ||
122 | |||
123 | path = d.getVar('PATH') | ||
124 | if path: | ||
125 | cmd = "PATH=\"%s\" %s" % (path, cmd) | ||
126 | bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) | ||
127 | |||
128 | ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) | ||
129 | |||
130 | os.chdir(save_cwd) | ||
131 | |||
132 | if ret != 0: | ||
133 | raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) | ||
134 | |||
135 | # if we have metadata to write out.. | ||
136 | if len(metadata) > 0: | ||
137 | cratepath = os.path.splitext(os.path.basename(thefile))[0] | ||
138 | bbpath = self._cargo_bitbake_path(rootdir) | ||
139 | mdfile = '.cargo-checksum.json' | ||
140 | mdpath = os.path.join(bbpath, cratepath, mdfile) | ||
141 | with open(mdpath, "w") as f: | ||
142 | json.dump(metadata, f) | ||
143 | |||
144 | def latest_versionstring(self, ud, d): | ||
145 | from functools import cmp_to_key | ||
146 | json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d)) | ||
147 | versions = [(0, i["num"], "") for i in json_data["versions"]] | ||
148 | versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) | ||
149 | |||
150 | return (versions[-1][1], "") | ||
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py new file mode 100644 index 0000000000..86546d40bf --- /dev/null +++ b/bitbake/lib/bb/fetch2/gcp.py | |||
@@ -0,0 +1,102 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' implementation for Google Cloup Platform Storage. | ||
3 | |||
4 | Class for fetching files from Google Cloud Storage using the | ||
5 | Google Cloud Storage Python Client. The GCS Python Client must | ||
6 | be correctly installed, configured and authenticated prior to use. | ||
7 | Additionally, gsutil must also be installed. | ||
8 | |||
9 | """ | ||
10 | |||
11 | # Copyright (C) 2023, Snap Inc. | ||
12 | # | ||
13 | # Based in part on bb.fetch2.s3: | ||
14 | # Copyright (C) 2017 Andre McCurdy | ||
15 | # | ||
16 | # SPDX-License-Identifier: GPL-2.0-only | ||
17 | # | ||
18 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
19 | |||
20 | import os | ||
21 | import bb | ||
22 | import urllib.parse, urllib.error | ||
23 | from bb.fetch2 import FetchMethod | ||
24 | from bb.fetch2 import FetchError | ||
25 | from bb.fetch2 import logger | ||
26 | |||
27 | class GCP(FetchMethod): | ||
28 | """ | ||
29 | Class to fetch urls via GCP's Python API. | ||
30 | """ | ||
31 | def __init__(self): | ||
32 | self.gcp_client = None | ||
33 | |||
34 | def supports(self, ud, d): | ||
35 | """ | ||
36 | Check to see if a given url can be fetched with GCP. | ||
37 | """ | ||
38 | return ud.type in ['gs'] | ||
39 | |||
40 | def recommends_checksum(self, urldata): | ||
41 | return True | ||
42 | |||
43 | def urldata_init(self, ud, d): | ||
44 | if 'downloadfilename' in ud.parm: | ||
45 | ud.basename = ud.parm['downloadfilename'] | ||
46 | else: | ||
47 | ud.basename = os.path.basename(ud.path) | ||
48 | |||
49 | ud.localfile = ud.basename | ||
50 | |||
51 | def get_gcp_client(self): | ||
52 | from google.cloud import storage | ||
53 | self.gcp_client = storage.Client(project=None) | ||
54 | |||
55 | def download(self, ud, d): | ||
56 | """ | ||
57 | Fetch urls using the GCP API. | ||
58 | Assumes localpath was called first. | ||
59 | """ | ||
60 | from google.api_core.exceptions import NotFound | ||
61 | logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}") | ||
62 | if self.gcp_client is None: | ||
63 | self.get_gcp_client() | ||
64 | |||
65 | bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}") | ||
66 | |||
67 | # Path sometimes has leading slash, so strip it | ||
68 | path = ud.path.lstrip("/") | ||
69 | blob = self.gcp_client.bucket(ud.host).blob(path) | ||
70 | try: | ||
71 | blob.download_to_filename(ud.localpath) | ||
72 | except NotFound: | ||
73 | raise FetchError("The GCP API threw a NotFound exception") | ||
74 | |||
75 | # Additional sanity checks copied from the wget class (although there | ||
76 | # are no known issues which mean these are required, treat the GCP API | ||
77 | # tool with a little healthy suspicion). | ||
78 | if not os.path.exists(ud.localpath): | ||
79 | raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!") | ||
80 | |||
81 | if os.path.getsize(ud.localpath) == 0: | ||
82 | os.remove(ud.localpath) | ||
83 | raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.") | ||
84 | |||
85 | return True | ||
86 | |||
87 | def checkstatus(self, fetch, ud, d): | ||
88 | """ | ||
89 | Check the status of a URL. | ||
90 | """ | ||
91 | logger.debug2(f"Checking status of gs://{ud.host}{ud.path}") | ||
92 | if self.gcp_client is None: | ||
93 | self.get_gcp_client() | ||
94 | |||
95 | bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}") | ||
96 | |||
97 | # Path sometimes has leading slash, so strip it | ||
98 | path = ud.path.lstrip("/") | ||
99 | if self.gcp_client.bucket(ud.host).blob(path).exists() == False: | ||
100 | raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist") | ||
101 | else: | ||
102 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index e3ba80a3f5..14ec45a3f6 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -9,15 +9,6 @@ Supported SRC_URI options are: | |||
9 | - branch | 9 | - branch |
10 | The git branch to retrieve from. The default is "master" | 10 | The git branch to retrieve from. The default is "master" |
11 | 11 | ||
12 | This option also supports multiple branch fetching, with branches | ||
13 | separated by commas. In multiple branches case, the name option | ||
14 | must have the same number of names to match the branches, which is | ||
15 | used to specify the SRC_REV for the branch | ||
16 | e.g: | ||
17 | SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" | ||
18 | SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" | ||
19 | SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" | ||
20 | |||
21 | - tag | 12 | - tag |
22 | The git tag to retrieve. The default is "master" | 13 | The git tag to retrieve. The default is "master" |
23 | 14 | ||
@@ -44,13 +35,27 @@ Supported SRC_URI options are: | |||
44 | 35 | ||
45 | - nobranch | 36 | - nobranch |
46 | Don't check the SHA validation for branch. set this option for the recipe | 37 | Don't check the SHA validation for branch. set this option for the recipe |
47 | referring to commit which is valid in tag instead of branch. | 38 | referring to commit which is valid in any namespace (branch, tag, ...) |
39 | instead of branch. | ||
48 | The default is "0", set nobranch=1 if needed. | 40 | The default is "0", set nobranch=1 if needed. |
49 | 41 | ||
42 | - subpath | ||
43 | Limit the checkout to a specific subpath of the tree. | ||
44 | By default, checkout the whole tree, set subpath=<path> if needed | ||
45 | |||
46 | - destsuffix | ||
47 | The name of the path in which to place the checkout. | ||
48 | By default, the path is git/, set destsuffix=<suffix> if needed | ||
49 | |||
50 | - usehead | 50 | - usehead |
51 | For local git:// urls to use the current branch HEAD as the revision for use with | 51 | For local git:// urls to use the current branch HEAD as the revision for use with |
52 | AUTOREV. Implies nobranch. | 52 | AUTOREV. Implies nobranch. |
53 | 53 | ||
54 | - lfs | ||
55 | Enable the checkout to use LFS for large files. This will download all LFS files | ||
56 | in the download step, as the unpack step does not have network access. | ||
57 | The default is "1", set lfs=0 to skip. | ||
58 | |||
54 | """ | 59 | """ |
55 | 60 | ||
56 | # Copyright (C) 2005 Richard Purdie | 61 | # Copyright (C) 2005 Richard Purdie |
@@ -64,15 +69,22 @@ import fnmatch | |||
64 | import os | 69 | import os |
65 | import re | 70 | import re |
66 | import shlex | 71 | import shlex |
72 | import shutil | ||
67 | import subprocess | 73 | import subprocess |
68 | import tempfile | 74 | import tempfile |
75 | import urllib | ||
69 | import bb | 76 | import bb |
70 | import bb.progress | 77 | import bb.progress |
78 | from contextlib import contextmanager | ||
71 | from bb.fetch2 import FetchMethod | 79 | from bb.fetch2 import FetchMethod |
72 | from bb.fetch2 import runfetchcmd | 80 | from bb.fetch2 import runfetchcmd |
73 | from bb.fetch2 import logger | 81 | from bb.fetch2 import logger |
82 | from bb.fetch2 import trusted_network | ||
74 | 83 | ||
75 | 84 | ||
85 | sha1_re = re.compile(r'^[0-9a-f]{40}$') | ||
86 | slash_re = re.compile(r"/+") | ||
87 | |||
76 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): | 88 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): |
77 | """Extract progress information from git output""" | 89 | """Extract progress information from git output""" |
78 | def __init__(self, d): | 90 | def __init__(self, d): |
@@ -130,6 +142,9 @@ class Git(FetchMethod): | |||
130 | def supports_checksum(self, urldata): | 142 | def supports_checksum(self, urldata): |
131 | return False | 143 | return False |
132 | 144 | ||
145 | def cleanup_upon_failure(self): | ||
146 | return False | ||
147 | |||
133 | def urldata_init(self, ud, d): | 148 | def urldata_init(self, ud, d): |
134 | """ | 149 | """ |
135 | init git specific variable within url data | 150 | init git specific variable within url data |
@@ -141,6 +156,11 @@ class Git(FetchMethod): | |||
141 | ud.proto = 'file' | 156 | ud.proto = 'file' |
142 | else: | 157 | else: |
143 | ud.proto = "git" | 158 | ud.proto = "git" |
159 | if ud.host == "github.com" and ud.proto == "git": | ||
160 | # github stopped supporting git protocol | ||
161 | # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git | ||
162 | ud.proto = "https" | ||
163 | bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url) | ||
144 | 164 | ||
145 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | 165 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
146 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | 166 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
@@ -162,18 +182,25 @@ class Git(FetchMethod): | |||
162 | ud.bareclone = ud.parm.get("bareclone","0") == "1" | 182 | ud.bareclone = ud.parm.get("bareclone","0") == "1" |
163 | if ud.bareclone: | 183 | if ud.bareclone: |
164 | ud.nocheckout = 1 | 184 | ud.nocheckout = 1 |
165 | |||
166 | ud.unresolvedrev = {} | ||
167 | branches = ud.parm.get("branch", "master").split(',') | ||
168 | if len(branches) != len(ud.names): | ||
169 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) | ||
170 | 185 | ||
171 | ud.cloneflags = "-s -n" | 186 | ud.unresolvedrev = "" |
187 | ud.branch = ud.parm.get("branch", "") | ||
188 | if not ud.branch and not ud.nobranch: | ||
189 | raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url) | ||
190 | |||
191 | ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" | ||
192 | |||
193 | ud.cloneflags = "-n" | ||
194 | if not ud.noshared: | ||
195 | ud.cloneflags += " -s" | ||
172 | if ud.bareclone: | 196 | if ud.bareclone: |
173 | ud.cloneflags += " --mirror" | 197 | ud.cloneflags += " --mirror" |
174 | 198 | ||
199 | ud.shallow_skip_fast = False | ||
175 | ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" | 200 | ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" |
176 | ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() | 201 | ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() |
202 | if 'tag' in ud.parm: | ||
203 | ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag']) | ||
177 | 204 | ||
178 | depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") | 205 | depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") |
179 | if depth_default is not None: | 206 | if depth_default is not None: |
@@ -190,32 +217,27 @@ class Git(FetchMethod): | |||
190 | 217 | ||
191 | revs_default = d.getVar("BB_GIT_SHALLOW_REVS") | 218 | revs_default = d.getVar("BB_GIT_SHALLOW_REVS") |
192 | ud.shallow_revs = [] | 219 | ud.shallow_revs = [] |
193 | ud.branches = {} | 220 | |
194 | for pos, name in enumerate(ud.names): | 221 | ud.unresolvedrev = ud.branch |
195 | branch = branches[pos] | 222 | |
196 | ud.branches[name] = branch | 223 | shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name) |
197 | ud.unresolvedrev[name] = branch | 224 | if shallow_depth is not None: |
198 | 225 | try: | |
199 | shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) | 226 | shallow_depth = int(shallow_depth or 0) |
200 | if shallow_depth is not None: | 227 | except ValueError: |
201 | try: | 228 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth)) |
202 | shallow_depth = int(shallow_depth or 0) | 229 | else: |
203 | except ValueError: | 230 | if shallow_depth < 0: |
204 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) | 231 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth)) |
205 | else: | 232 | ud.shallow_depths[ud.name] = shallow_depth |
206 | if shallow_depth < 0: | 233 | |
207 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) | 234 | revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name) |
208 | ud.shallow_depths[name] = shallow_depth | 235 | if revs is not None: |
209 | 236 | ud.shallow_revs.extend(revs.split()) | |
210 | revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) | 237 | elif revs_default is not None: |
211 | if revs is not None: | 238 | ud.shallow_revs.extend(revs_default.split()) |
212 | ud.shallow_revs.extend(revs.split()) | 239 | |
213 | elif revs_default is not None: | 240 | if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0: |
214 | ud.shallow_revs.extend(revs_default.split()) | ||
215 | |||
216 | if (ud.shallow and | ||
217 | not ud.shallow_revs and | ||
218 | all(ud.shallow_depths[n] == 0 for n in ud.names)): | ||
219 | # Shallow disabled for this URL | 241 | # Shallow disabled for this URL |
220 | ud.shallow = False | 242 | ud.shallow = False |
221 | 243 | ||
@@ -224,10 +246,9 @@ class Git(FetchMethod): | |||
224 | # rev of this repository. This will get resolved into a revision | 246 | # rev of this repository. This will get resolved into a revision |
225 | # later. If an actual revision happens to have also been provided | 247 | # later. If an actual revision happens to have also been provided |
226 | # then this setting will be overridden. | 248 | # then this setting will be overridden. |
227 | for name in ud.names: | 249 | ud.unresolvedrev = 'HEAD' |
228 | ud.unresolvedrev[name] = 'HEAD' | ||
229 | 250 | ||
230 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" | 251 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin" |
231 | 252 | ||
232 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" | 253 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" |
233 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable | 254 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable |
@@ -235,24 +256,22 @@ class Git(FetchMethod): | |||
235 | 256 | ||
236 | ud.setup_revisions(d) | 257 | ud.setup_revisions(d) |
237 | 258 | ||
238 | for name in ud.names: | 259 | # Ensure any revision that doesn't look like a SHA-1 is translated into one |
239 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one | 260 | if not sha1_re.match(ud.revision or ''): |
240 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): | 261 | if ud.revision: |
241 | if ud.revisions[name]: | 262 | ud.unresolvedrev = ud.revision |
242 | ud.unresolvedrev[name] = ud.revisions[name] | 263 | ud.revision = self.latest_revision(ud, d, ud.name) |
243 | ud.revisions[name] = self.latest_revision(ud, d, name) | ||
244 | 264 | ||
245 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) | 265 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) |
246 | if gitsrcname.startswith('.'): | 266 | if gitsrcname.startswith('.'): |
247 | gitsrcname = gitsrcname[1:] | 267 | gitsrcname = gitsrcname[1:] |
248 | 268 | ||
249 | # for rebaseable git repo, it is necessary to keep mirror tar ball | 269 | # For a rebaseable git repo, it is necessary to keep a mirror tar ball |
250 | # per revision, so that even the revision disappears from the | 270 | # per revision, so that even if the revision disappears from the |
251 | # upstream repo in the future, the mirror will remain intact and still | 271 | # upstream repo in the future, the mirror will remain intact and still |
252 | # contains the revision | 272 | # contain the revision |
253 | if ud.rebaseable: | 273 | if ud.rebaseable: |
254 | for name in ud.names: | 274 | gitsrcname = gitsrcname + '_' + ud.revision |
255 | gitsrcname = gitsrcname + '_' + ud.revisions[name] | ||
256 | 275 | ||
257 | dl_dir = d.getVar("DL_DIR") | 276 | dl_dir = d.getVar("DL_DIR") |
258 | gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") | 277 | gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") |
@@ -270,15 +289,14 @@ class Git(FetchMethod): | |||
270 | if ud.shallow_revs: | 289 | if ud.shallow_revs: |
271 | tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) | 290 | tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) |
272 | 291 | ||
273 | for name, revision in sorted(ud.revisions.items()): | 292 | tarballname = "%s_%s" % (tarballname, ud.revision[:7]) |
274 | tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) | 293 | depth = ud.shallow_depths[ud.name] |
275 | depth = ud.shallow_depths[name] | 294 | if depth: |
276 | if depth: | 295 | tarballname = "%s-%s" % (tarballname, depth) |
277 | tarballname = "%s-%s" % (tarballname, depth) | ||
278 | 296 | ||
279 | shallow_refs = [] | 297 | shallow_refs = [] |
280 | if not ud.nobranch: | 298 | if not ud.nobranch: |
281 | shallow_refs.extend(ud.branches.values()) | 299 | shallow_refs.append(ud.branch) |
282 | if ud.shallow_extra_refs: | 300 | if ud.shallow_extra_refs: |
283 | shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) | 301 | shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) |
284 | if shallow_refs: | 302 | if shallow_refs: |
@@ -293,16 +311,29 @@ class Git(FetchMethod): | |||
293 | return ud.clonedir | 311 | return ud.clonedir |
294 | 312 | ||
295 | def need_update(self, ud, d): | 313 | def need_update(self, ud, d): |
296 | return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) | 314 | return self.clonedir_need_update(ud, d) \ |
315 | or self.shallow_tarball_need_update(ud) \ | ||
316 | or self.tarball_need_update(ud) \ | ||
317 | or self.lfs_need_update(ud, d) | ||
297 | 318 | ||
298 | def clonedir_need_update(self, ud, d): | 319 | def clonedir_need_update(self, ud, d): |
299 | if not os.path.exists(ud.clonedir): | 320 | if not os.path.exists(ud.clonedir): |
300 | return True | 321 | return True |
301 | if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): | 322 | if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): |
302 | return True | 323 | return True |
303 | for name in ud.names: | 324 | if not self._contains_ref(ud, d, ud.name, ud.clonedir): |
304 | if not self._contains_ref(ud, d, name, ud.clonedir): | 325 | return True |
305 | return True | 326 | return False |
327 | |||
328 | def lfs_need_update(self, ud, d): | ||
329 | if not self._need_lfs(ud): | ||
330 | return False | ||
331 | |||
332 | if self.clonedir_need_update(ud, d): | ||
333 | return True | ||
334 | |||
335 | if not self._lfs_objects_downloaded(ud, d, ud.clonedir): | ||
336 | return True | ||
306 | return False | 337 | return False |
307 | 338 | ||
308 | def clonedir_need_shallow_revs(self, ud, d): | 339 | def clonedir_need_shallow_revs(self, ud, d): |
@@ -319,11 +350,28 @@ class Git(FetchMethod): | |||
319 | def tarball_need_update(self, ud): | 350 | def tarball_need_update(self, ud): |
320 | return ud.write_tarballs and not os.path.exists(ud.fullmirror) | 351 | return ud.write_tarballs and not os.path.exists(ud.fullmirror) |
321 | 352 | ||
353 | def update_mirror_links(self, ud, origud): | ||
354 | super().update_mirror_links(ud, origud) | ||
355 | # When using shallow mode, add a symlink to the original fullshallow | ||
356 | # path to ensure a valid symlink even in the `PREMIRRORS` case | ||
357 | if ud.shallow and not os.path.exists(origud.fullshallow): | ||
358 | self.ensure_symlink(ud.localpath, origud.fullshallow) | ||
359 | |||
322 | def try_premirror(self, ud, d): | 360 | def try_premirror(self, ud, d): |
323 | # If we don't do this, updating an existing checkout with only premirrors | 361 | # If we don't do this, updating an existing checkout with only premirrors |
324 | # is not possible | 362 | # is not possible |
325 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): | 363 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): |
326 | return True | 364 | return True |
365 | # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0 | ||
366 | # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then | ||
367 | # we need to try premirrors first as using upstream is destined to fail. | ||
368 | if not trusted_network(d, ud.url): | ||
369 | return True | ||
370 | # the following check is to ensure incremental fetch in downloads, this is | ||
371 | # because the premirror might be old and does not contain the new rev required, | ||
372 | # and this will cause a total removal and new clone. So if we can reach to | ||
373 | # network, we prefer upstream over premirror, though the premirror might contain | ||
374 | # the new rev. | ||
327 | if os.path.exists(ud.clonedir): | 375 | if os.path.exists(ud.clonedir): |
328 | return False | 376 | return False |
329 | return True | 377 | return True |
@@ -337,21 +385,76 @@ class Git(FetchMethod): | |||
337 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | 385 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): |
338 | ud.localpath = ud.fullshallow | 386 | ud.localpath = ud.fullshallow |
339 | return | 387 | return |
340 | elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): | 388 | elif os.path.exists(ud.fullmirror) and self.need_update(ud, d): |
341 | bb.utils.mkdirhier(ud.clonedir) | 389 | if not os.path.exists(ud.clonedir): |
342 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | 390 | bb.utils.mkdirhier(ud.clonedir) |
343 | 391 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | |
392 | else: | ||
393 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
394 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) | ||
395 | output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) | ||
396 | if 'mirror' in output: | ||
397 | runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) | ||
398 | runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) | ||
399 | fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) | ||
400 | runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) | ||
344 | repourl = self._get_repo_url(ud) | 401 | repourl = self._get_repo_url(ud) |
345 | 402 | ||
403 | needs_clone = False | ||
404 | if os.path.exists(ud.clonedir): | ||
405 | # The directory may exist, but not be the top level of a bare git | ||
406 | # repository in which case it needs to be deleted and re-cloned. | ||
407 | try: | ||
408 | # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel | ||
409 | output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) | ||
410 | toplevel = output.rstrip() | ||
411 | |||
412 | if not bb.utils.path_is_descendant(toplevel, ud.clonedir): | ||
413 | logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir) | ||
414 | needs_clone = True | ||
415 | except bb.fetch2.FetchError as e: | ||
416 | logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) | ||
417 | needs_clone = True | ||
418 | except FileNotFoundError as e: | ||
419 | logger.warning("%s", e) | ||
420 | needs_clone = True | ||
421 | |||
422 | if needs_clone: | ||
423 | shutil.rmtree(ud.clonedir) | ||
424 | else: | ||
425 | needs_clone = True | ||
426 | |||
346 | # If the repo still doesn't exist, fallback to cloning it | 427 | # If the repo still doesn't exist, fallback to cloning it |
347 | if not os.path.exists(ud.clonedir): | 428 | if needs_clone: |
348 | # We do this since git will use a "-l" option automatically for local urls where possible | 429 | # We do this since git will use a "-l" option automatically for local urls where possible, |
430 | # but it doesn't work when git/objects is a symlink, only works when it is a directory. | ||
349 | if repourl.startswith("file://"): | 431 | if repourl.startswith("file://"): |
350 | repourl = repourl[7:] | 432 | repourl_path = repourl[7:] |
433 | objects = os.path.join(repourl_path, 'objects') | ||
434 | if os.path.isdir(objects) and not os.path.islink(objects): | ||
435 | repourl = repourl_path | ||
351 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) | 436 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) |
352 | if ud.proto.lower() != 'file': | 437 | if ud.proto.lower() != 'file': |
353 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) | 438 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) |
354 | progresshandler = GitProgressHandler(d) | 439 | progresshandler = GitProgressHandler(d) |
440 | |||
441 | # Try creating a fast initial shallow clone | ||
442 | # Enabling ud.shallow_skip_fast will skip this | ||
443 | # If the Git error "Server does not allow request for unadvertised object" | ||
444 | # occurs, shallow_skip_fast is enabled automatically. | ||
445 | # This may happen if the Git server does not allow the request | ||
446 | # or if the Git client has issues with this functionality. | ||
447 | if ud.shallow and not ud.shallow_skip_fast: | ||
448 | try: | ||
449 | self.clone_shallow_with_tarball(ud, d) | ||
450 | # When the shallow clone has succeeded, use the shallow tarball | ||
451 | ud.localpath = ud.fullshallow | ||
452 | return | ||
453 | except: | ||
454 | logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.") | ||
455 | |||
456 | # When skipping fast initial shallow or the fast inital shallow clone failed: | ||
457 | # Try again with an initial regular clone | ||
355 | runfetchcmd(clone_cmd, d, log=progresshandler) | 458 | runfetchcmd(clone_cmd, d, log=progresshandler) |
356 | 459 | ||
357 | # Update the checkout if needed | 460 | # Update the checkout if needed |
@@ -361,7 +464,11 @@ class Git(FetchMethod): | |||
361 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) | 464 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
362 | 465 | ||
363 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) | 466 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) |
364 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | 467 | |
468 | if ud.nobranch: | ||
469 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | ||
470 | else: | ||
471 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) | ||
365 | if ud.proto.lower() != 'file': | 472 | if ud.proto.lower() != 'file': |
366 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | 473 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
367 | progresshandler = GitProgressHandler(d) | 474 | progresshandler = GitProgressHandler(d) |
@@ -375,138 +482,206 @@ class Git(FetchMethod): | |||
375 | if exc.errno != errno.ENOENT: | 482 | if exc.errno != errno.ENOENT: |
376 | raise | 483 | raise |
377 | 484 | ||
378 | for name in ud.names: | 485 | if not self._contains_ref(ud, d, ud.name, ud.clonedir): |
379 | if not self._contains_ref(ud, d, name, ud.clonedir): | 486 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch)) |
380 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) | ||
381 | 487 | ||
382 | if ud.shallow and ud.write_shallow_tarballs: | 488 | if ud.shallow and ud.write_shallow_tarballs: |
383 | missing_rev = self.clonedir_need_shallow_revs(ud, d) | 489 | missing_rev = self.clonedir_need_shallow_revs(ud, d) |
384 | if missing_rev: | 490 | if missing_rev: |
385 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 491 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
386 | 492 | ||
387 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | 493 | if self.lfs_need_update(ud, d): |
388 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | 494 | self.lfs_fetch(ud, d, ud.clonedir, ud.revision) |
389 | # of all LFS blobs needed at the the srcrev. | 495 | |
390 | # | 496 | def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None): |
391 | # It would be nice to just do this inline here by running 'git-lfs fetch' | 497 | """Helper method for fetching Git LFS data""" |
392 | # on the bare clonedir, but that operation requires a working copy on some | 498 | try: |
393 | # releases of Git LFS. | 499 | if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision): |
394 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 500 | self._ensure_git_lfs(d, ud) |
395 | try: | 501 | |
396 | # Do the checkout. This implicitly involves a Git LFS fetch. | 502 | # Using worktree with the revision because .lfsconfig may exists |
397 | self.unpack(ud, tmpdir, d) | 503 | worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision) |
398 | 504 | runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir) | |
399 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | 505 | lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "") |
400 | # the bare clonedir. | 506 | runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt")) |
401 | # | 507 | worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd |
402 | # As this procedure is invoked repeatedly on incremental fetches as | 508 | runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir) |
403 | # a recipe's SRCREV is bumped throughout its lifetime, this will | 509 | except: |
404 | # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs | 510 | logger.warning("Fetching LFS did not succeed.") |
405 | # corresponding to all the blobs reachable from the different revs | 511 | |
406 | # fetched across time. | 512 | @contextmanager |
407 | # | 513 | def create_atomic(self, filename): |
408 | # Only do this if the unpack resulted in a .git/lfs directory being | 514 | """Create as a temp file and move atomically into position to avoid races""" |
409 | # created; this only happens if at least one blob needed to be | 515 | fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) |
410 | # downloaded. | 516 | try: |
411 | if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): | 517 | yield tfile |
412 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) | 518 | umask = os.umask(0o666) |
413 | finally: | 519 | os.umask(umask) |
414 | bb.utils.remove(tmpdir, recurse=True) | 520 | os.chmod(tfile, (0o666 & ~umask)) |
521 | os.rename(tfile, filename) | ||
522 | finally: | ||
523 | os.close(fd) | ||
415 | 524 | ||
416 | def build_mirror_data(self, ud, d): | 525 | def build_mirror_data(self, ud, d): |
417 | if ud.shallow and ud.write_shallow_tarballs: | 526 | if ud.shallow and ud.write_shallow_tarballs: |
418 | if not os.path.exists(ud.fullshallow): | 527 | if not os.path.exists(ud.fullshallow): |
419 | if os.path.islink(ud.fullshallow): | 528 | if os.path.islink(ud.fullshallow): |
420 | os.unlink(ud.fullshallow) | 529 | os.unlink(ud.fullshallow) |
421 | tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 530 | self.clone_shallow_with_tarball(ud, d) |
422 | shallowclone = os.path.join(tempdir, 'git') | ||
423 | try: | ||
424 | self.clone_shallow_local(ud, shallowclone, d) | ||
425 | |||
426 | logger.info("Creating tarball of git repository") | ||
427 | runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) | ||
428 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | ||
429 | finally: | ||
430 | bb.utils.remove(tempdir, recurse=True) | ||
431 | elif ud.write_tarballs and not os.path.exists(ud.fullmirror): | 531 | elif ud.write_tarballs and not os.path.exists(ud.fullmirror): |
432 | if os.path.islink(ud.fullmirror): | 532 | if os.path.islink(ud.fullmirror): |
433 | os.unlink(ud.fullmirror) | 533 | os.unlink(ud.fullmirror) |
434 | 534 | ||
435 | logger.info("Creating tarball of git repository") | 535 | logger.info("Creating tarball of git repository") |
436 | runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) | 536 | with self.create_atomic(ud.fullmirror) as tfile: |
537 | mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d, | ||
538 | quiet=True, workdir=ud.clonedir) | ||
539 | runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ." | ||
540 | % (tfile, mtime), d, workdir=ud.clonedir) | ||
437 | runfetchcmd("touch %s.done" % ud.fullmirror, d) | 541 | runfetchcmd("touch %s.done" % ud.fullmirror, d) |
438 | 542 | ||
543 | def clone_shallow_with_tarball(self, ud, d): | ||
544 | ret = False | ||
545 | tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
546 | shallowclone = os.path.join(tempdir, 'git') | ||
547 | try: | ||
548 | try: | ||
549 | self.clone_shallow_local(ud, shallowclone, d) | ||
550 | except: | ||
551 | logger.warning("Fast shallow clone failed, try to skip fast mode now.") | ||
552 | bb.utils.remove(tempdir, recurse=True) | ||
553 | os.mkdir(tempdir) | ||
554 | ud.shallow_skip_fast = True | ||
555 | self.clone_shallow_local(ud, shallowclone, d) | ||
556 | logger.info("Creating tarball of git repository") | ||
557 | with self.create_atomic(ud.fullshallow) as tfile: | ||
558 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) | ||
559 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | ||
560 | ret = True | ||
561 | finally: | ||
562 | bb.utils.remove(tempdir, recurse=True) | ||
563 | |||
564 | return ret | ||
565 | |||
439 | def clone_shallow_local(self, ud, dest, d): | 566 | def clone_shallow_local(self, ud, dest, d): |
440 | """Clone the repo and make it shallow. | 567 | """ |
568 | Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default): | ||
569 | - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev | ||
570 | - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev | ||
571 | """ | ||
441 | 572 | ||
442 | The upstream url of the new clone isn't set at this time, as it'll be | 573 | progresshandler = GitProgressHandler(d) |
443 | set correctly when unpacked.""" | 574 | repourl = self._get_repo_url(ud) |
444 | runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) | 575 | bb.utils.mkdirhier(dest) |
576 | init_cmd = "%s init -q" % ud.basecmd | ||
577 | if ud.bareclone: | ||
578 | init_cmd += " --bare" | ||
579 | runfetchcmd(init_cmd, d, workdir=dest) | ||
580 | # Use repourl when creating a fast initial shallow clone | ||
581 | # Prefer already existing full bare clones if available | ||
582 | if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir): | ||
583 | remote = shlex.quote(repourl) | ||
584 | else: | ||
585 | remote = ud.clonedir | ||
586 | runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest) | ||
445 | 587 | ||
446 | to_parse, shallow_branches = [], [] | 588 | # Check the histories which should be excluded |
447 | for name in ud.names: | 589 | shallow_exclude = '' |
448 | revision = ud.revisions[name] | 590 | for revision in ud.shallow_revs: |
449 | depth = ud.shallow_depths[name] | 591 | shallow_exclude += " --shallow-exclude=%s" % revision |
450 | if depth: | ||
451 | to_parse.append('%s~%d^{}' % (revision, depth - 1)) | ||
452 | 592 | ||
453 | # For nobranch, we need a ref, otherwise the commits will be | 593 | revision = ud.revision |
454 | # removed, and for non-nobranch, we truncate the branch to our | 594 | depth = ud.shallow_depths[ud.name] |
455 | # srcrev, to avoid keeping unnecessary history beyond that. | ||
456 | branch = ud.branches[name] | ||
457 | if ud.nobranch: | ||
458 | ref = "refs/shallow/%s" % name | ||
459 | elif ud.bareclone: | ||
460 | ref = "refs/heads/%s" % branch | ||
461 | else: | ||
462 | ref = "refs/remotes/origin/%s" % branch | ||
463 | 595 | ||
464 | shallow_branches.append(ref) | 596 | # The --depth and --shallow-exclude can't be used together |
465 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | 597 | if depth and shallow_exclude: |
598 | raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.") | ||
466 | 599 | ||
467 | # Map srcrev+depths to revisions | 600 | # For nobranch, we need a ref, otherwise the commits will be |
468 | parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) | 601 | # removed, and for non-nobranch, we truncate the branch to our |
602 | # srcrev, to avoid keeping unnecessary history beyond that. | ||
603 | branch = ud.branch | ||
604 | if ud.nobranch: | ||
605 | ref = "refs/shallow/%s" % ud.name | ||
606 | elif ud.bareclone: | ||
607 | ref = "refs/heads/%s" % branch | ||
608 | else: | ||
609 | ref = "refs/remotes/origin/%s" % branch | ||
610 | |||
611 | fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision) | ||
612 | if depth: | ||
613 | fetch_cmd += " --depth %s" % depth | ||
614 | |||
615 | if shallow_exclude: | ||
616 | fetch_cmd += shallow_exclude | ||
469 | 617 | ||
470 | # Resolve specified revisions | 618 | # Advertise the revision for lower version git such as 2.25.1: |
471 | parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) | 619 | # error: Server does not allow request for unadvertised object. |
472 | shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() | 620 | # The ud.clonedir is a local temporary dir, will be removed when |
621 | # fetch is done, so we can do anything on it. | ||
622 | adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision) | ||
623 | if ud.shallow_skip_fast: | ||
624 | runfetchcmd(adv_cmd, d, workdir=ud.clonedir) | ||
625 | |||
626 | runfetchcmd(fetch_cmd, d, workdir=dest) | ||
627 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | ||
628 | # Fetch Git LFS data | ||
629 | self.lfs_fetch(ud, d, dest, ud.revision) | ||
473 | 630 | ||
474 | # Apply extra ref wildcards | 631 | # Apply extra ref wildcards |
475 | all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, | 632 | all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \ |
476 | d, workdir=dest).splitlines() | 633 | d, workdir=dest).splitlines() |
634 | all_refs = [] | ||
635 | for line in all_refs_remote: | ||
636 | all_refs.append(line.split()[-1]) | ||
637 | extra_refs = [] | ||
477 | for r in ud.shallow_extra_refs: | 638 | for r in ud.shallow_extra_refs: |
478 | if not ud.bareclone: | 639 | if not ud.bareclone: |
479 | r = r.replace('refs/heads/', 'refs/remotes/origin/') | 640 | r = r.replace('refs/heads/', 'refs/remotes/origin/') |
480 | 641 | ||
481 | if '*' in r: | 642 | if '*' in r: |
482 | matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) | 643 | matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) |
483 | shallow_branches.extend(matches) | 644 | extra_refs.extend(matches) |
484 | else: | 645 | else: |
485 | shallow_branches.append(r) | 646 | extra_refs.append(r) |
647 | |||
648 | for ref in extra_refs: | ||
649 | ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '') | ||
650 | runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest) | ||
651 | revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest) | ||
652 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | ||
486 | 653 | ||
487 | # Make the repository shallow | 654 | # The url is local ud.clonedir, set it to upstream one |
488 | shallow_cmd = [self.make_shallow_path, '-s'] | 655 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest) |
489 | for b in shallow_branches: | ||
490 | shallow_cmd.append('-r') | ||
491 | shallow_cmd.append(b) | ||
492 | shallow_cmd.extend(shallow_revisions) | ||
493 | runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest) | ||
494 | 656 | ||
495 | def unpack(self, ud, destdir, d): | 657 | def unpack(self, ud, destdir, d): |
496 | """ unpack the downloaded src to destdir""" | 658 | """ unpack the downloaded src to destdir""" |
497 | 659 | ||
498 | subdir = ud.parm.get("subpath", "") | 660 | subdir = ud.parm.get("subdir") |
499 | if subdir != "": | 661 | subpath = ud.parm.get("subpath") |
500 | readpathspec = ":%s" % subdir | 662 | readpathspec = "" |
501 | def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) | 663 | def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/" |
502 | else: | 664 | |
503 | readpathspec = "" | 665 | if subpath: |
504 | def_destsuffix = "git/" | 666 | readpathspec = ":%s" % subpath |
667 | def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/')) | ||
668 | |||
669 | if subdir: | ||
670 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | ||
671 | if os.path.isabs(subdir): | ||
672 | if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)): | ||
673 | raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url) | ||
674 | destdir = subdir | ||
675 | else: | ||
676 | destdir = os.path.join(destdir, subdir) | ||
677 | def_destsuffix = "" | ||
505 | 678 | ||
506 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) | 679 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) |
507 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | 680 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
508 | if os.path.exists(destdir): | 681 | if os.path.exists(destdir): |
509 | bb.utils.prunedir(destdir) | 682 | bb.utils.prunedir(destdir) |
683 | if not ud.bareclone: | ||
684 | ud.unpack_tracer.unpack("git", destdir) | ||
510 | 685 | ||
511 | need_lfs = self._need_lfs(ud) | 686 | need_lfs = self._need_lfs(ud) |
512 | 687 | ||
@@ -516,13 +691,12 @@ class Git(FetchMethod): | |||
516 | source_found = False | 691 | source_found = False |
517 | source_error = [] | 692 | source_error = [] |
518 | 693 | ||
519 | if not source_found: | 694 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) |
520 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) | 695 | if clonedir_is_up_to_date: |
521 | if clonedir_is_up_to_date: | 696 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) |
522 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) | 697 | source_found = True |
523 | source_found = True | 698 | else: |
524 | else: | 699 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) |
525 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) | ||
526 | 700 | ||
527 | if not source_found: | 701 | if not source_found: |
528 | if ud.shallow: | 702 | if ud.shallow: |
@@ -538,28 +712,43 @@ class Git(FetchMethod): | |||
538 | if not source_found: | 712 | if not source_found: |
539 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) | 713 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) |
540 | 714 | ||
715 | # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag | ||
716 | # matches the revision | ||
717 | if 'tag' in ud.parm and sha1_re.match(ud.revision): | ||
718 | output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir) | ||
719 | output = output.strip() | ||
720 | if output != ud.revision: | ||
721 | # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list | ||
722 | # If it resolves to the same thing there isn't a problem. | ||
723 | output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir) | ||
724 | output2 = output2.strip() | ||
725 | if output != output2: | ||
726 | raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url) | ||
727 | |||
541 | repourl = self._get_repo_url(ud) | 728 | repourl = self._get_repo_url(ud) |
542 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) | 729 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) |
543 | 730 | ||
544 | if self._contains_lfs(ud, d, destdir): | 731 | if self._contains_lfs(ud, d, destdir): |
545 | if need_lfs and not self._find_git_lfs(d): | 732 | if not need_lfs: |
546 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) | ||
547 | elif not need_lfs: | ||
548 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) | 733 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) |
734 | else: | ||
735 | self._ensure_git_lfs(d, ud) | ||
736 | |||
737 | runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) | ||
549 | 738 | ||
550 | if not ud.nocheckout: | 739 | if not ud.nocheckout: |
551 | if subdir != "": | 740 | if subpath: |
552 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, | 741 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d, |
553 | workdir=destdir) | 742 | workdir=destdir) |
554 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) | 743 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) |
555 | elif not ud.nobranch: | 744 | elif not ud.nobranch: |
556 | branchname = ud.branches[ud.names[0]] | 745 | branchname = ud.branch |
557 | runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ | 746 | runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ |
558 | ud.revisions[ud.names[0]]), d, workdir=destdir) | 747 | ud.revision), d, workdir=destdir) |
559 | runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ | 748 | runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ |
560 | branchname), d, workdir=destdir) | 749 | branchname), d, workdir=destdir) |
561 | else: | 750 | else: |
562 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) | 751 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir) |
563 | 752 | ||
564 | return True | 753 | return True |
565 | 754 | ||
@@ -573,8 +762,13 @@ class Git(FetchMethod): | |||
573 | clonedir = os.path.realpath(ud.localpath) | 762 | clonedir = os.path.realpath(ud.localpath) |
574 | to_remove.append(clonedir) | 763 | to_remove.append(clonedir) |
575 | 764 | ||
765 | # Remove shallow mirror tarball | ||
766 | if ud.shallow: | ||
767 | to_remove.append(ud.fullshallow) | ||
768 | to_remove.append(ud.fullshallow + ".done") | ||
769 | |||
576 | for r in to_remove: | 770 | for r in to_remove: |
577 | if os.path.exists(r): | 771 | if os.path.exists(r) or os.path.islink(r): |
578 | bb.note('Removing %s' % r) | 772 | bb.note('Removing %s' % r) |
579 | bb.utils.remove(r, True) | 773 | bb.utils.remove(r, True) |
580 | 774 | ||
@@ -585,10 +779,10 @@ class Git(FetchMethod): | |||
585 | cmd = "" | 779 | cmd = "" |
586 | if ud.nobranch: | 780 | if ud.nobranch: |
587 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( | 781 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( |
588 | ud.basecmd, ud.revisions[name]) | 782 | ud.basecmd, ud.revision) |
589 | else: | 783 | else: |
590 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( | 784 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( |
591 | ud.basecmd, ud.revisions[name], ud.branches[name]) | 785 | ud.basecmd, ud.revision, ud.branch) |
592 | try: | 786 | try: |
593 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) | 787 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
594 | except bb.fetch2.FetchError: | 788 | except bb.fetch2.FetchError: |
@@ -597,6 +791,37 @@ class Git(FetchMethod): | |||
597 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 791 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
598 | return output.split()[0] != "0" | 792 | return output.split()[0] != "0" |
599 | 793 | ||
794 | def _lfs_objects_downloaded(self, ud, d, wd): | ||
795 | """ | ||
796 | Verifies whether the LFS objects for requested revisions have already been downloaded | ||
797 | """ | ||
798 | # Bail out early if this repository doesn't use LFS | ||
799 | if not self._contains_lfs(ud, d, wd): | ||
800 | return True | ||
801 | |||
802 | self._ensure_git_lfs(d, ud) | ||
803 | |||
804 | # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file | ||
805 | # existence. | ||
806 | # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git | ||
807 | cmd = "%s lfs ls-files -l %s" \ | ||
808 | % (ud.basecmd, ud.revision) | ||
809 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() | ||
810 | # Do not do any further matching if no objects are managed by LFS | ||
811 | if not output: | ||
812 | return True | ||
813 | |||
814 | # Match all lines beginning with the hexadecimal OID | ||
815 | oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") | ||
816 | for line in output.split("\n"): | ||
817 | oid = re.search(oid_regex, line) | ||
818 | if not oid: | ||
819 | bb.warn("git lfs ls-files output '%s' did not match expected format." % line) | ||
820 | if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): | ||
821 | return False | ||
822 | |||
823 | return True | ||
824 | |||
600 | def _need_lfs(self, ud): | 825 | def _need_lfs(self, ud): |
601 | return ud.parm.get("lfs", "1") == "1" | 826 | return ud.parm.get("lfs", "1") == "1" |
602 | 827 | ||
@@ -604,20 +829,8 @@ class Git(FetchMethod): | |||
604 | """ | 829 | """ |
605 | Check if the repository has 'lfs' (large file) content | 830 | Check if the repository has 'lfs' (large file) content |
606 | """ | 831 | """ |
607 | |||
608 | if not ud.nobranch: | ||
609 | branchname = ud.branches[ud.names[0]] | ||
610 | else: | ||
611 | branchname = "master" | ||
612 | |||
613 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | ||
614 | if wd == ud.clonedir: | ||
615 | refname = ud.branches[ud.names[0]] | ||
616 | else: | ||
617 | refname = "origin/%s" % ud.branches[ud.names[0]] | ||
618 | |||
619 | cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( | 832 | cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( |
620 | ud.basecmd, refname) | 833 | ud.basecmd, ud.revision) |
621 | 834 | ||
622 | try: | 835 | try: |
623 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) | 836 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
@@ -627,12 +840,14 @@ class Git(FetchMethod): | |||
627 | pass | 840 | pass |
628 | return False | 841 | return False |
629 | 842 | ||
630 | def _find_git_lfs(self, d): | 843 | def _ensure_git_lfs(self, d, ud): |
631 | """ | 844 | """ |
632 | Return True if git-lfs can be found, False otherwise. | 845 | Ensures that git-lfs is available, raising a FetchError if it isn't. |
633 | """ | 846 | """ |
634 | import shutil | 847 | if shutil.which("git-lfs", path=d.getVar('PATH')) is None: |
635 | return shutil.which("git-lfs", path=d.getVar('PATH')) is not None | 848 | raise bb.fetch2.FetchError( |
849 | "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 " | ||
850 | "to ignore it)" % self._get_repo_url(ud)) | ||
636 | 851 | ||
637 | def _get_repo_url(self, ud): | 852 | def _get_repo_url(self, ud): |
638 | """ | 853 | """ |
@@ -640,22 +855,21 @@ class Git(FetchMethod): | |||
640 | """ | 855 | """ |
641 | # Note that we do not support passwords directly in the git urls. There are several | 856 | # Note that we do not support passwords directly in the git urls. There are several |
642 | # reasons. SRC_URI can be written out to things like buildhistory and people don't | 857 | # reasons. SRC_URI can be written out to things like buildhistory and people don't |
643 | # want to leak passwords like that. Its also all too easy to share metadata without | 858 | # want to leak passwords like that. Its also all too easy to share metadata without |
644 | # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as | 859 | # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as |
645 | # alternatives so we will not take patches adding password support here. | 860 | # alternatives so we will not take patches adding password support here. |
646 | if ud.user: | 861 | if ud.user: |
647 | username = ud.user + '@' | 862 | username = ud.user + '@' |
648 | else: | 863 | else: |
649 | username = "" | 864 | username = "" |
650 | return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) | 865 | return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path)) |
651 | 866 | ||
652 | def _revision_key(self, ud, d, name): | 867 | def _revision_key(self, ud, d, name): |
653 | """ | 868 | """ |
654 | Return a unique key for the url | 869 | Return a unique key for the url |
655 | """ | 870 | """ |
656 | # Collapse adjacent slashes | 871 | # Collapse adjacent slashes |
657 | slash_re = re.compile(r"/+") | 872 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev |
658 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] | ||
659 | 873 | ||
660 | def _lsremote(self, ud, d, search): | 874 | def _lsremote(self, ud, d, search): |
661 | """ | 875 | """ |
@@ -687,21 +901,27 @@ class Git(FetchMethod): | |||
687 | """ | 901 | """ |
688 | Compute the HEAD revision for the url | 902 | Compute the HEAD revision for the url |
689 | """ | 903 | """ |
904 | if not d.getVar("__BBSRCREV_SEEN"): | ||
905 | raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path)) | ||
906 | |||
907 | # Ensure we mark as not cached | ||
908 | bb.fetch2.mark_recipe_nocache(d) | ||
909 | |||
690 | output = self._lsremote(ud, d, "") | 910 | output = self._lsremote(ud, d, "") |
691 | # Tags of the form ^{} may not work, need to fallback to other form | 911 | # Tags of the form ^{} may not work, need to fallback to other form |
692 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: | 912 | if ud.unresolvedrev[:5] == "refs/" or ud.usehead: |
693 | head = ud.unresolvedrev[name] | 913 | head = ud.unresolvedrev |
694 | tag = ud.unresolvedrev[name] | 914 | tag = ud.unresolvedrev |
695 | else: | 915 | else: |
696 | head = "refs/heads/%s" % ud.unresolvedrev[name] | 916 | head = "refs/heads/%s" % ud.unresolvedrev |
697 | tag = "refs/tags/%s" % ud.unresolvedrev[name] | 917 | tag = "refs/tags/%s" % ud.unresolvedrev |
698 | for s in [head, tag + "^{}", tag]: | 918 | for s in [head, tag + "^{}", tag]: |
699 | for l in output.strip().split('\n'): | 919 | for l in output.strip().split('\n'): |
700 | sha1, ref = l.split() | 920 | sha1, ref = l.split() |
701 | if s == ref: | 921 | if s == ref: |
702 | return sha1 | 922 | return sha1 |
703 | raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ | 923 | raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ |
704 | (ud.unresolvedrev[name], ud.host+ud.path)) | 924 | (ud.unresolvedrev, ud.host+ud.path)) |
705 | 925 | ||
706 | def latest_versionstring(self, ud, d): | 926 | def latest_versionstring(self, ud, d): |
707 | """ | 927 | """ |
@@ -711,60 +931,63 @@ class Git(FetchMethod): | |||
711 | """ | 931 | """ |
712 | pupver = ('', '') | 932 | pupver = ('', '') |
713 | 933 | ||
714 | tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
715 | try: | 934 | try: |
716 | output = self._lsremote(ud, d, "refs/tags/*") | 935 | output = self._lsremote(ud, d, "refs/tags/*") |
717 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: | 936 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: |
718 | bb.note("Could not list remote: %s" % str(e)) | 937 | bb.note("Could not list remote: %s" % str(e)) |
719 | return pupver | 938 | return pupver |
720 | 939 | ||
940 | rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)") | ||
941 | pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
942 | nonrel_re = re.compile(r"(alpha|beta|rc|final)+") | ||
943 | |||
721 | verstring = "" | 944 | verstring = "" |
722 | revision = "" | ||
723 | for line in output.split("\n"): | 945 | for line in output.split("\n"): |
724 | if not line: | 946 | if not line: |
725 | break | 947 | break |
726 | 948 | ||
727 | tag_head = line.split("/")[-1] | 949 | m = rev_tag_re.match(line) |
950 | if not m: | ||
951 | continue | ||
952 | |||
953 | (revision, tag) = m.groups() | ||
954 | |||
728 | # Ignore non-released branches | 955 | # Ignore non-released branches |
729 | m = re.search(r"(alpha|beta|rc|final)+", tag_head) | 956 | if nonrel_re.search(tag): |
730 | if m: | ||
731 | continue | 957 | continue |
732 | 958 | ||
733 | # search for version in the line | 959 | # search for version in the line |
734 | tag = tagregex.search(tag_head) | 960 | m = pver_re.search(tag) |
735 | if tag is None: | 961 | if not m: |
736 | continue | 962 | continue |
737 | 963 | ||
738 | tag = tag.group('pver') | 964 | pver = m.group('pver').replace("_", ".") |
739 | tag = tag.replace("_", ".") | ||
740 | 965 | ||
741 | if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: | 966 | if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0: |
742 | continue | 967 | continue |
743 | 968 | ||
744 | verstring = tag | 969 | verstring = pver |
745 | revision = line.split()[0] | ||
746 | pupver = (verstring, revision) | 970 | pupver = (verstring, revision) |
747 | 971 | ||
748 | return pupver | 972 | return pupver |
749 | 973 | ||
750 | def _build_revision(self, ud, d, name): | 974 | def _build_revision(self, ud, d, name): |
751 | return ud.revisions[name] | 975 | return ud.revision |
752 | 976 | ||
753 | def gitpkgv_revision(self, ud, d, name): | 977 | def gitpkgv_revision(self, ud, d, name): |
754 | """ | 978 | """ |
755 | Return a sortable revision number by counting commits in the history | 979 | Return a sortable revision number by counting commits in the history |
756 | Based on gitpkgv.bblass in meta-openembedded | 980 | Based on gitpkgv.bblass in meta-openembedded |
757 | """ | 981 | """ |
758 | rev = self._build_revision(ud, d, name) | 982 | rev = ud.revision |
759 | localpath = ud.localpath | 983 | localpath = ud.localpath |
760 | rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) | 984 | rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) |
761 | if not os.path.exists(localpath): | 985 | if not os.path.exists(localpath): |
762 | commits = None | 986 | commits = None |
763 | else: | 987 | else: |
764 | if not os.path.exists(rev_file) or not os.path.getsize(rev_file): | 988 | if not os.path.exists(rev_file) or not os.path.getsize(rev_file): |
765 | from pipes import quote | ||
766 | commits = bb.fetch2.runfetchcmd( | 989 | commits = bb.fetch2.runfetchcmd( |
767 | "git rev-list %s -- | wc -l" % quote(rev), | 990 | "git rev-list %s -- | wc -l" % shlex.quote(rev), |
768 | d, quiet=True).strip().lstrip('0') | 991 | d, quiet=True).strip().lstrip('0') |
769 | if commits: | 992 | if commits: |
770 | open(rev_file, "w").write("%d\n" % int(commits)) | 993 | open(rev_file, "w").write("%d\n" % int(commits)) |
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index a4527bf364..5869e1b99b 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py | |||
@@ -62,36 +62,35 @@ class GitSM(Git): | |||
62 | return modules | 62 | return modules |
63 | 63 | ||
64 | # Collect the defined submodules, and their attributes | 64 | # Collect the defined submodules, and their attributes |
65 | for name in ud.names: | 65 | try: |
66 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir) | ||
67 | except: | ||
68 | # No submodules to update | ||
69 | gitmodules = "" | ||
70 | |||
71 | for m, md in parse_gitmodules(gitmodules).items(): | ||
66 | try: | 72 | try: |
67 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) | 73 | module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir) |
68 | except: | 74 | except: |
69 | # No submodules to update | 75 | # If the command fails, we don't have a valid file to check. If it doesn't |
76 | # fail -- it still might be a failure, see next check... | ||
77 | module_hash = "" | ||
78 | |||
79 | if not module_hash: | ||
80 | logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) | ||
70 | continue | 81 | continue |
71 | 82 | ||
72 | for m, md in parse_gitmodules(gitmodules).items(): | 83 | submodules.append(m) |
73 | try: | 84 | paths[m] = md['path'] |
74 | module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) | 85 | revision[m] = ud.revision |
75 | except: | 86 | uris[m] = md['url'] |
76 | # If the command fails, we don't have a valid file to check. If it doesn't | 87 | subrevision[m] = module_hash.split()[2] |
77 | # fail -- it still might be a failure, see next check... | 88 | |
78 | module_hash = "" | 89 | # Convert relative to absolute uri based on parent uri |
79 | 90 | if uris[m].startswith('..') or uris[m].startswith('./'): | |
80 | if not module_hash: | 91 | newud = copy.copy(ud) |
81 | logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) | 92 | newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) |
82 | continue | 93 | uris[m] = Git._get_repo_url(self, newud) |
83 | |||
84 | submodules.append(m) | ||
85 | paths[m] = md['path'] | ||
86 | revision[m] = ud.revisions[name] | ||
87 | uris[m] = md['url'] | ||
88 | subrevision[m] = module_hash.split()[2] | ||
89 | |||
90 | # Convert relative to absolute uri based on parent uri | ||
91 | if uris[m].startswith('..'): | ||
92 | newud = copy.copy(ud) | ||
93 | newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) | ||
94 | uris[m] = Git._get_repo_url(self, newud) | ||
95 | 94 | ||
96 | for module in submodules: | 95 | for module in submodules: |
97 | # Translate the module url into a SRC_URI | 96 | # Translate the module url into a SRC_URI |
@@ -115,10 +114,21 @@ class GitSM(Git): | |||
115 | # This has to be a file reference | 114 | # This has to be a file reference |
116 | proto = "file" | 115 | proto = "file" |
117 | url = "gitsm://" + uris[module] | 116 | url = "gitsm://" + uris[module] |
117 | if url.endswith("{}{}".format(ud.host, ud.path)): | ||
118 | raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ | ||
119 | "Consider using git fetcher instead.") | ||
118 | 120 | ||
119 | url += ';protocol=%s' % proto | 121 | url += ';protocol=%s' % proto |
120 | url += ";name=%s" % module | 122 | url += ";name=%s" % module |
121 | url += ";subpath=%s" % module | 123 | url += ";subpath=%s" % module |
124 | url += ";nobranch=1" | ||
125 | url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0") | ||
126 | # Note that adding "user=" here to give credentials to the | ||
127 | # submodule is not supported. Since using SRC_URI to give git:// | ||
128 | # URL a password is not supported, one have to use one of the | ||
129 | # recommended way (eg. ~/.netrc or SSH config) which does specify | ||
130 | # the user (See comment in git.py). | ||
131 | # So, we will not take patches adding "user=" support here. | ||
122 | 132 | ||
123 | ld = d.createCopy() | 133 | ld = d.createCopy() |
124 | # Not necessary to set SRC_URI, since we're passing the URI to | 134 | # Not necessary to set SRC_URI, since we're passing the URI to |
@@ -136,20 +146,26 @@ class GitSM(Git): | |||
136 | 146 | ||
137 | return submodules != [] | 147 | return submodules != [] |
138 | 148 | ||
149 | def call_process_submodules(self, ud, d, extra_check, subfunc): | ||
150 | # If we're using a shallow mirror tarball it needs to be | ||
151 | # unpacked temporarily so that we can examine the .gitmodules file | ||
152 | # Unpack even when ud.clonedir is not available, | ||
153 | # which may occur during a fast shallow clone | ||
154 | unpack = extra_check or not os.path.exists(ud.clonedir) | ||
155 | if ud.shallow and os.path.exists(ud.fullshallow) and unpack: | ||
156 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
157 | try: | ||
158 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
159 | self.process_submodules(ud, tmpdir, subfunc, d) | ||
160 | finally: | ||
161 | shutil.rmtree(tmpdir) | ||
162 | else: | ||
163 | self.process_submodules(ud, ud.clonedir, subfunc, d) | ||
164 | |||
139 | def need_update(self, ud, d): | 165 | def need_update(self, ud, d): |
140 | if Git.need_update(self, ud, d): | 166 | if Git.need_update(self, ud, d): |
141 | return True | 167 | return True |
142 | 168 | ||
143 | try: | ||
144 | # Check for the nugget dropped by the download operation | ||
145 | known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ | ||
146 | (ud.basecmd), d, workdir=ud.clonedir) | ||
147 | |||
148 | if ud.revisions[ud.names[0]] in known_srcrevs.split(): | ||
149 | return False | ||
150 | except bb.fetch2.FetchError: | ||
151 | pass | ||
152 | |||
153 | need_update_list = [] | 169 | need_update_list = [] |
154 | def need_update_submodule(ud, url, module, modpath, workdir, d): | 170 | def need_update_submodule(ud, url, module, modpath, workdir, d): |
155 | url += ";bareclone=1;nobranch=1" | 171 | url += ";bareclone=1;nobranch=1" |
@@ -163,22 +179,9 @@ class GitSM(Git): | |||
163 | logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) | 179 | logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) |
164 | need_update_result = True | 180 | need_update_result = True |
165 | 181 | ||
166 | # If we're using a shallow mirror tarball it needs to be unpacked | 182 | self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule) |
167 | # temporarily so that we can examine the .gitmodules file | 183 | |
168 | if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): | 184 | if need_update_list: |
169 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
170 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
171 | self.process_submodules(ud, tmpdir, need_update_submodule, d) | ||
172 | shutil.rmtree(tmpdir) | ||
173 | else: | ||
174 | self.process_submodules(ud, ud.clonedir, need_update_submodule, d) | ||
175 | if len(need_update_list) == 0: | ||
176 | # We already have the required commits of all submodules. Drop | ||
177 | # a nugget so we don't need to check again. | ||
178 | runfetchcmd("%s config --add bitbake.srcrev %s" % \ | ||
179 | (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) | ||
180 | |||
181 | if len(need_update_list) > 0: | ||
182 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) | 185 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) |
183 | return True | 186 | return True |
184 | 187 | ||
@@ -199,19 +202,7 @@ class GitSM(Git): | |||
199 | raise | 202 | raise |
200 | 203 | ||
201 | Git.download(self, ud, d) | 204 | Git.download(self, ud, d) |
202 | 205 | self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule) | |
203 | # If we're using a shallow mirror tarball it needs to be unpacked | ||
204 | # temporarily so that we can examine the .gitmodules file | ||
205 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | ||
206 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
207 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
208 | self.process_submodules(ud, tmpdir, download_submodule, d) | ||
209 | shutil.rmtree(tmpdir) | ||
210 | else: | ||
211 | self.process_submodules(ud, ud.clonedir, download_submodule, d) | ||
212 | # Drop a nugget for the srcrev we've fetched (used by need_update) | ||
213 | runfetchcmd("%s config --add bitbake.srcrev %s" % \ | ||
214 | (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) | ||
215 | 206 | ||
216 | def unpack(self, ud, destdir, d): | 207 | def unpack(self, ud, destdir, d): |
217 | def unpack_submodules(ud, url, module, modpath, workdir, d): | 208 | def unpack_submodules(ud, url, module, modpath, workdir, d): |
@@ -225,6 +216,10 @@ class GitSM(Git): | |||
225 | 216 | ||
226 | try: | 217 | try: |
227 | newfetch = Fetch([url], d, cache=False) | 218 | newfetch = Fetch([url], d, cache=False) |
219 | # modpath is needed by unpack tracer to calculate submodule | ||
220 | # checkout dir | ||
221 | new_ud = newfetch.ud[url] | ||
222 | new_ud.modpath = modpath | ||
228 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) | 223 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) |
229 | except Exception as e: | 224 | except Exception as e: |
230 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) | 225 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) |
@@ -250,13 +245,27 @@ class GitSM(Git): | |||
250 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) | 245 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) |
251 | 246 | ||
252 | if not ud.bareclone and ret: | 247 | if not ud.bareclone and ret: |
253 | # All submodules should already be downloaded and configured in the tree. This simply sets | 248 | cmdprefix = "" |
254 | # up the configuration and checks out the files. The main project config should remain | 249 | # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed. |
255 | # unmodified, and no download from the internet should occur. | 250 | if not self._need_lfs(ud): |
256 | runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) | 251 | cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 " |
252 | runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir) | ||
253 | def clean(self, ud, d): | ||
254 | def clean_submodule(ud, url, module, modpath, workdir, d): | ||
255 | url += ";bareclone=1;nobranch=1" | ||
256 | try: | ||
257 | newfetch = Fetch([url], d, cache=False) | ||
258 | newfetch.clean() | ||
259 | except Exception as e: | ||
260 | logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e))) | ||
261 | |||
262 | self.call_process_submodules(ud, d, True, clean_submodule) | ||
263 | |||
264 | # Clean top git dir | ||
265 | Git.clean(self, ud, d) | ||
257 | 266 | ||
258 | def implicit_urldata(self, ud, d): | 267 | def implicit_urldata(self, ud, d): |
259 | import shutil, subprocess, tempfile | 268 | import subprocess |
260 | 269 | ||
261 | urldata = [] | 270 | urldata = [] |
262 | def add_submodule(ud, url, module, modpath, workdir, d): | 271 | def add_submodule(ud, url, module, modpath, workdir, d): |
@@ -264,14 +273,6 @@ class GitSM(Git): | |||
264 | newfetch = Fetch([url], d, cache=False) | 273 | newfetch = Fetch([url], d, cache=False) |
265 | urldata.extend(newfetch.expanded_urldata()) | 274 | urldata.extend(newfetch.expanded_urldata()) |
266 | 275 | ||
267 | # If we're using a shallow mirror tarball it needs to be unpacked | 276 | self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule) |
268 | # temporarily so that we can examine the .gitmodules file | ||
269 | if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): | ||
270 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
271 | subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) | ||
272 | self.process_submodules(ud, tmpdir, add_submodule, d) | ||
273 | shutil.rmtree(tmpdir) | ||
274 | else: | ||
275 | self.process_submodules(ud, ud.clonedir, add_submodule, d) | ||
276 | 277 | ||
277 | return urldata | 278 | return urldata |
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py new file mode 100644 index 0000000000..53c1d8d115 --- /dev/null +++ b/bitbake/lib/bb/fetch2/gomod.py | |||
@@ -0,0 +1,273 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' implementation for Go modules | ||
3 | |||
4 | The gomod/gomodgit fetchers are used to download Go modules to the module cache | ||
5 | from a module proxy or directly from a version control repository. | ||
6 | |||
7 | Example SRC_URI: | ||
8 | |||
9 | SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..." | ||
10 | SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..." | ||
11 | |||
12 | Required SRC_URI parameters: | ||
13 | |||
14 | - version | ||
15 | The version of the module. | ||
16 | |||
17 | Optional SRC_URI parameters: | ||
18 | |||
19 | - mod | ||
20 | Fetch and unpack the go.mod file only instead of the complete module. | ||
21 | The go command may need to download go.mod files for many different modules | ||
22 | when computing the build list, and go.mod files are much smaller than | ||
23 | module zip files. | ||
24 | The default is "0", set mod=1 for the go.mod file only. | ||
25 | |||
26 | - sha256sum | ||
27 | The checksum of the module zip file, or the go.mod file in case of fetching | ||
28 | only the go.mod file. Alternatively, set the SRC_URI varible flag for | ||
29 | "module@version.sha256sum". | ||
30 | |||
31 | - protocol | ||
32 | The method used when fetching directly from a version control repository. | ||
33 | The default is "https" for git. | ||
34 | |||
35 | - repo | ||
36 | The URL when fetching directly from a version control repository. Required | ||
37 | when the URL is different from the module path. | ||
38 | |||
39 | - srcrev | ||
40 | The revision identifier used when fetching directly from a version control | ||
41 | repository. Alternatively, set the SRCREV varible for "module@version". | ||
42 | |||
43 | - subdir | ||
44 | The module subdirectory when fetching directly from a version control | ||
45 | repository. Required when the module is not located in the root of the | ||
46 | repository. | ||
47 | |||
48 | Related variables: | ||
49 | |||
50 | - GO_MOD_PROXY | ||
51 | The module proxy used by the fetcher. | ||
52 | |||
53 | - GO_MOD_CACHE_DIR | ||
54 | The directory where the module cache is located. | ||
55 | This must match the exported GOMODCACHE variable for the go command to find | ||
56 | the downloaded modules. | ||
57 | |||
58 | See the Go modules reference, https://go.dev/ref/mod, for more information | ||
59 | about the module cache, module proxies and version control systems. | ||
60 | """ | ||
61 | |||
62 | import hashlib | ||
63 | import os | ||
64 | import re | ||
65 | import shutil | ||
66 | import subprocess | ||
67 | import zipfile | ||
68 | |||
69 | import bb | ||
70 | from bb.fetch2 import FetchError | ||
71 | from bb.fetch2 import MissingParameterError | ||
72 | from bb.fetch2 import runfetchcmd | ||
73 | from bb.fetch2 import subprocess_setup | ||
74 | from bb.fetch2.git import Git | ||
75 | from bb.fetch2.wget import Wget | ||
76 | |||
77 | |||
78 | def escape(path): | ||
79 | """Escape capital letters using exclamation points.""" | ||
80 | return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path) | ||
81 | |||
82 | |||
83 | class GoMod(Wget): | ||
84 | """Class to fetch Go modules from a Go module proxy via wget""" | ||
85 | |||
86 | def supports(self, ud, d): | ||
87 | """Check to see if a given URL is for this fetcher.""" | ||
88 | return ud.type == 'gomod' | ||
89 | |||
90 | def urldata_init(self, ud, d): | ||
91 | """Set up to download the module from the module proxy. | ||
92 | |||
93 | Set up to download the module zip file to the module cache directory | ||
94 | and unpack the go.mod file (unless downloading only the go.mod file): | ||
95 | |||
96 | cache/download/<module>/@v/<version>.zip: The module zip file. | ||
97 | cache/download/<module>/@v/<version>.mod: The go.mod file. | ||
98 | """ | ||
99 | |||
100 | proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org' | ||
101 | moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' | ||
102 | |||
103 | if 'version' not in ud.parm: | ||
104 | raise MissingParameterError('version', ud.url) | ||
105 | |||
106 | module = ud.host | ||
107 | if ud.path != '/': | ||
108 | module += ud.path | ||
109 | ud.parm['module'] = module | ||
110 | version = ud.parm['version'] | ||
111 | |||
112 | # Set URL and filename for wget download | ||
113 | if ud.parm.get('mod', '0') == '1': | ||
114 | ext = '.mod' | ||
115 | else: | ||
116 | ext = '.zip' | ||
117 | path = escape(f"{module}/@v/{version}{ext}") | ||
118 | ud.url = bb.fetch2.encodeurl( | ||
119 | ('https', proxy, '/' + path, None, None, None)) | ||
120 | ud.parm['downloadfilename'] = f"{module.replace('/', '.')}@{version}{ext}" | ||
121 | |||
122 | # Set name for checksum verification | ||
123 | ud.parm['name'] = f"{module}@{version}" | ||
124 | |||
125 | # Set path for unpack | ||
126 | ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path) | ||
127 | |||
128 | super().urldata_init(ud, d) | ||
129 | |||
130 | def unpack(self, ud, rootdir, d): | ||
131 | """Unpack the module in the module cache.""" | ||
132 | |||
133 | # Unpack the module zip file or go.mod file | ||
134 | unpackpath = os.path.join(rootdir, ud.parm['unpackpath']) | ||
135 | unpackdir = os.path.dirname(unpackpath) | ||
136 | bb.utils.mkdirhier(unpackdir) | ||
137 | ud.unpack_tracer.unpack("file-copy", unpackdir) | ||
138 | cmd = f"cp {ud.localpath} {unpackpath}" | ||
139 | path = d.getVar('PATH') | ||
140 | if path: | ||
141 | cmd = f"PATH={path} {cmd}" | ||
142 | name = os.path.basename(unpackpath) | ||
143 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
144 | subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup) | ||
145 | |||
146 | if name.endswith('.zip'): | ||
147 | # Unpack the go.mod file from the zip file | ||
148 | module = ud.parm['module'] | ||
149 | name = name.rsplit('.', 1)[0] + '.mod' | ||
150 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
151 | with zipfile.ZipFile(ud.localpath) as zf: | ||
152 | with open(os.path.join(unpackdir, name), mode='wb') as mf: | ||
153 | try: | ||
154 | f = module + '@' + ud.parm['version'] + '/go.mod' | ||
155 | shutil.copyfileobj(zf.open(f), mf) | ||
156 | except KeyError: | ||
157 | # If the module does not have a go.mod file, synthesize | ||
158 | # one containing only a module statement. | ||
159 | mf.write(f'module {module}\n'.encode()) | ||
160 | |||
161 | |||
162 | class GoModGit(Git): | ||
163 | """Class to fetch Go modules directly from a git repository""" | ||
164 | |||
165 | def supports(self, ud, d): | ||
166 | """Check to see if a given URL is for this fetcher.""" | ||
167 | return ud.type == 'gomodgit' | ||
168 | |||
169 | def urldata_init(self, ud, d): | ||
170 | """Set up to download the module from the git repository. | ||
171 | |||
172 | Set up to download the git repository to the module cache directory and | ||
173 | unpack the module zip file and the go.mod file: | ||
174 | |||
175 | cache/vcs/<hash>: The bare git repository. | ||
176 | cache/download/<module>/@v/<version>.zip: The module zip file. | ||
177 | cache/download/<module>/@v/<version>.mod: The go.mod file. | ||
178 | """ | ||
179 | |||
180 | moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' | ||
181 | |||
182 | if 'version' not in ud.parm: | ||
183 | raise MissingParameterError('version', ud.url) | ||
184 | |||
185 | module = ud.host | ||
186 | if ud.path != '/': | ||
187 | module += ud.path | ||
188 | ud.parm['module'] = module | ||
189 | |||
190 | # Set host, path and srcrev for git download | ||
191 | if 'repo' in ud.parm: | ||
192 | repo = ud.parm['repo'] | ||
193 | idx = repo.find('/') | ||
194 | if idx != -1: | ||
195 | ud.host = repo[:idx] | ||
196 | ud.path = repo[idx:] | ||
197 | else: | ||
198 | ud.host = repo | ||
199 | ud.path = '' | ||
200 | if 'protocol' not in ud.parm: | ||
201 | ud.parm['protocol'] = 'https' | ||
202 | ud.name = f"{module}@{ud.parm['version']}" | ||
203 | srcrev = d.getVar('SRCREV_' + ud.name) | ||
204 | if srcrev: | ||
205 | if 'srcrev' not in ud.parm: | ||
206 | ud.parm['srcrev'] = srcrev | ||
207 | else: | ||
208 | if 'srcrev' in ud.parm: | ||
209 | d.setVar('SRCREV_' + ud.name, ud.parm['srcrev']) | ||
210 | if 'branch' not in ud.parm: | ||
211 | ud.parm['nobranch'] = '1' | ||
212 | |||
213 | # Set subpath, subdir and bareclone for git unpack | ||
214 | if 'subdir' in ud.parm: | ||
215 | ud.parm['subpath'] = ud.parm['subdir'] | ||
216 | key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode() | ||
217 | ud.parm['key'] = key | ||
218 | ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs', | ||
219 | hashlib.sha256(key).hexdigest()) | ||
220 | ud.parm['bareclone'] = '1' | ||
221 | |||
222 | super().urldata_init(ud, d) | ||
223 | |||
224 | def unpack(self, ud, rootdir, d): | ||
225 | """Unpack the module in the module cache.""" | ||
226 | |||
227 | # Unpack the bare git repository | ||
228 | super().unpack(ud, rootdir, d) | ||
229 | |||
230 | moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' | ||
231 | |||
232 | # Create the info file | ||
233 | module = ud.parm['module'] | ||
234 | repodir = os.path.join(rootdir, ud.parm['subdir']) | ||
235 | with open(repodir + '.info', 'wb') as f: | ||
236 | f.write(ud.parm['key']) | ||
237 | |||
238 | # Unpack the go.mod file from the repository | ||
239 | unpackdir = os.path.join(rootdir, moddir, 'cache/download', | ||
240 | escape(module), '@v') | ||
241 | bb.utils.mkdirhier(unpackdir) | ||
242 | srcrev = ud.parm['srcrev'] | ||
243 | version = ud.parm['version'] | ||
244 | escaped_version = escape(version) | ||
245 | cmd = f"git ls-tree -r --name-only '{srcrev}'" | ||
246 | if 'subpath' in ud.parm: | ||
247 | cmd += f" '{ud.parm['subpath']}'" | ||
248 | files = runfetchcmd(cmd, d, workdir=repodir).split() | ||
249 | name = escaped_version + '.mod' | ||
250 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
251 | with open(os.path.join(unpackdir, name), mode='wb') as mf: | ||
252 | f = 'go.mod' | ||
253 | if 'subpath' in ud.parm: | ||
254 | f = os.path.join(ud.parm['subpath'], f) | ||
255 | if f in files: | ||
256 | cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] | ||
257 | subprocess.check_call(cmd, stdout=mf, cwd=repodir, | ||
258 | preexec_fn=subprocess_setup) | ||
259 | else: | ||
260 | # If the module does not have a go.mod file, synthesize one | ||
261 | # containing only a module statement. | ||
262 | mf.write(f'module {module}\n'.encode()) | ||
263 | |||
264 | # Synthesize the module zip file from the repository | ||
265 | name = escaped_version + '.zip' | ||
266 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
267 | with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf: | ||
268 | prefix = module + '@' + version + '/' | ||
269 | for f in files: | ||
270 | cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] | ||
271 | data = subprocess.check_output(cmd, cwd=repodir, | ||
272 | preexec_fn=subprocess_setup) | ||
273 | zf.writestr(prefix + f, data) | ||
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index 063e13008a..cbff8c490c 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py | |||
@@ -242,6 +242,7 @@ class Hg(FetchMethod): | |||
242 | revflag = "-r %s" % ud.revision | 242 | revflag = "-r %s" % ud.revision |
243 | subdir = ud.parm.get("destsuffix", ud.module) | 243 | subdir = ud.parm.get("destsuffix", ud.module) |
244 | codir = "%s/%s" % (destdir, subdir) | 244 | codir = "%s/%s" % (destdir, subdir) |
245 | ud.unpack_tracer.unpack("hg", codir) | ||
245 | 246 | ||
246 | scmdata = ud.parm.get("scmdata", "") | 247 | scmdata = ud.parm.get("scmdata", "") |
247 | if scmdata != "nokeep": | 248 | if scmdata != "nokeep": |
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py index e7d1c8c58f..fda56a564e 100644 --- a/bitbake/lib/bb/fetch2/local.py +++ b/bitbake/lib/bb/fetch2/local.py | |||
@@ -29,11 +29,10 @@ class Local(FetchMethod): | |||
29 | 29 | ||
30 | def urldata_init(self, ud, d): | 30 | def urldata_init(self, ud, d): |
31 | # We don't set localfile as for this fetcher the file is already local! | 31 | # We don't set localfile as for this fetcher the file is already local! |
32 | ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0]) | 32 | ud.basename = os.path.basename(ud.path) |
33 | ud.basename = os.path.basename(ud.decodedurl) | 33 | ud.basepath = ud.path |
34 | ud.basepath = ud.decodedurl | ||
35 | ud.needdonestamp = False | 34 | ud.needdonestamp = False |
36 | if "*" in ud.decodedurl: | 35 | if "*" in ud.path: |
37 | raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url) | 36 | raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url) |
38 | return | 37 | return |
39 | 38 | ||
@@ -41,28 +40,24 @@ class Local(FetchMethod): | |||
41 | """ | 40 | """ |
42 | Return the local filename of a given url assuming a successful fetch. | 41 | Return the local filename of a given url assuming a successful fetch. |
43 | """ | 42 | """ |
44 | return self.localpaths(urldata, d)[-1] | 43 | return self.localfile_searchpaths(urldata, d)[-1] |
45 | 44 | ||
46 | def localpaths(self, urldata, d): | 45 | def localfile_searchpaths(self, urldata, d): |
47 | """ | 46 | """ |
48 | Return the local filename of a given url assuming a successful fetch. | 47 | Return the local filename of a given url assuming a successful fetch. |
49 | """ | 48 | """ |
50 | searched = [] | 49 | searched = [] |
51 | path = urldata.decodedurl | 50 | path = urldata.path |
52 | newpath = path | 51 | newpath = path |
53 | if path[0] == "/": | 52 | if path[0] == "/": |
53 | logger.debug2("Using absolute %s" % (path)) | ||
54 | return [path] | 54 | return [path] |
55 | filespath = d.getVar('FILESPATH') | 55 | filespath = d.getVar('FILESPATH') |
56 | if filespath: | 56 | if filespath: |
57 | logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) | 57 | logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) |
58 | newpath, hist = bb.utils.which(filespath, path, history=True) | 58 | newpath, hist = bb.utils.which(filespath, path, history=True) |
59 | logger.debug2("Using %s for %s" % (newpath, path)) | ||
59 | searched.extend(hist) | 60 | searched.extend(hist) |
60 | if not os.path.exists(newpath): | ||
61 | dldirfile = os.path.join(d.getVar("DL_DIR"), path) | ||
62 | logger.debug2("Defaulting to %s for %s" % (dldirfile, path)) | ||
63 | bb.utils.mkdirhier(os.path.dirname(dldirfile)) | ||
64 | searched.append(dldirfile) | ||
65 | return searched | ||
66 | return searched | 61 | return searched |
67 | 62 | ||
68 | def need_update(self, ud, d): | 63 | def need_update(self, ud, d): |
@@ -78,9 +73,7 @@ class Local(FetchMethod): | |||
78 | filespath = d.getVar('FILESPATH') | 73 | filespath = d.getVar('FILESPATH') |
79 | if filespath: | 74 | if filespath: |
80 | locations = filespath.split(":") | 75 | locations = filespath.split(":") |
81 | locations.append(d.getVar("DL_DIR")) | 76 | msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n " + "\n ".join(locations) |
82 | |||
83 | msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations) | ||
84 | raise FetchError(msg) | 77 | raise FetchError(msg) |
85 | 78 | ||
86 | return True | 79 | return True |
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index 47898509ff..e469d66768 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py | |||
@@ -42,19 +42,27 @@ from bb.utils import is_semver | |||
42 | 42 | ||
43 | def npm_package(package): | 43 | def npm_package(package): |
44 | """Convert the npm package name to remove unsupported character""" | 44 | """Convert the npm package name to remove unsupported character""" |
45 | # Scoped package names (with the @) use the same naming convention | 45 | # For scoped package names ('@user/package') the '/' is replaced by a '-'. |
46 | # as the 'npm pack' command. | 46 | # This is similar to what 'npm pack' does, but 'npm pack' also strips the |
47 | if package.startswith("@"): | 47 | # leading '@', which can lead to ambiguous package names. |
48 | return re.sub("/", "-", package[1:]) | 48 | name = re.sub("/", "-", package) |
49 | return package | 49 | name = name.lower() |
50 | name = re.sub(r"[^\-a-z0-9@]", "", name) | ||
51 | name = name.strip("-") | ||
52 | return name | ||
53 | |||
50 | 54 | ||
51 | def npm_filename(package, version): | 55 | def npm_filename(package, version): |
52 | """Get the filename of a npm package""" | 56 | """Get the filename of a npm package""" |
53 | return npm_package(package) + "-" + version + ".tgz" | 57 | return npm_package(package) + "-" + version + ".tgz" |
54 | 58 | ||
55 | def npm_localfile(package, version): | 59 | def npm_localfile(package, version=None): |
56 | """Get the local filename of a npm package""" | 60 | """Get the local filename of a npm package""" |
57 | return os.path.join("npm2", npm_filename(package, version)) | 61 | if version is not None: |
62 | filename = npm_filename(package, version) | ||
63 | else: | ||
64 | filename = package | ||
65 | return os.path.join("npm2", filename) | ||
58 | 66 | ||
59 | def npm_integrity(integrity): | 67 | def npm_integrity(integrity): |
60 | """ | 68 | """ |
@@ -69,41 +77,67 @@ def npm_unpack(tarball, destdir, d): | |||
69 | bb.utils.mkdirhier(destdir) | 77 | bb.utils.mkdirhier(destdir) |
70 | cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) | 78 | cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) |
71 | cmd += " --no-same-owner" | 79 | cmd += " --no-same-owner" |
80 | cmd += " --delay-directory-restore" | ||
72 | cmd += " --strip-components=1" | 81 | cmd += " --strip-components=1" |
73 | runfetchcmd(cmd, d, workdir=destdir) | 82 | runfetchcmd(cmd, d, workdir=destdir) |
83 | runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) | ||
74 | 84 | ||
75 | class NpmEnvironment(object): | 85 | class NpmEnvironment(object): |
76 | """ | 86 | """ |
77 | Using a npm config file seems more reliable than using cli arguments. | 87 | Using a npm config file seems more reliable than using cli arguments. |
78 | This class allows to create a controlled environment for npm commands. | 88 | This class allows to create a controlled environment for npm commands. |
79 | """ | 89 | """ |
80 | def __init__(self, d, configs=None): | 90 | def __init__(self, d, configs=[], npmrc=None): |
81 | self.d = d | 91 | self.d = d |
82 | self.configs = configs | 92 | |
93 | self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) | ||
94 | |||
95 | hn = self._home_npmrc(d) | ||
96 | if hn is not None: | ||
97 | with open(hn, 'r') as hnf: | ||
98 | self.user_config.write(hnf.read()) | ||
99 | |||
100 | for key, value in configs: | ||
101 | self.user_config.write("%s=%s\n" % (key, value)) | ||
102 | |||
103 | if npmrc: | ||
104 | self.global_config_name = npmrc | ||
105 | else: | ||
106 | self.global_config_name = "/dev/null" | ||
107 | |||
108 | def __del__(self): | ||
109 | if self.user_config: | ||
110 | self.user_config.close() | ||
111 | |||
112 | def _home_npmrc(self, d): | ||
113 | """Function to return user's HOME .npmrc file (or None if it doesn't exist)""" | ||
114 | home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc") | ||
115 | if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file): | ||
116 | bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\ | ||
117 | f"npm fetcher will use {home_npmrc_file}") | ||
118 | return home_npmrc_file | ||
119 | return None | ||
83 | 120 | ||
84 | def run(self, cmd, args=None, configs=None, workdir=None): | 121 | def run(self, cmd, args=None, configs=None, workdir=None): |
85 | """Run npm command in a controlled environment""" | 122 | """Run npm command in a controlled environment""" |
86 | with tempfile.TemporaryDirectory() as tmpdir: | 123 | with tempfile.TemporaryDirectory() as tmpdir: |
87 | d = bb.data.createCopy(self.d) | 124 | d = bb.data.createCopy(self.d) |
125 | d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching | ||
88 | d.setVar("HOME", tmpdir) | 126 | d.setVar("HOME", tmpdir) |
89 | 127 | ||
90 | cfgfile = os.path.join(tmpdir, "npmrc") | ||
91 | |||
92 | if not workdir: | 128 | if not workdir: |
93 | workdir = tmpdir | 129 | workdir = tmpdir |
94 | 130 | ||
95 | def _run(cmd): | 131 | def _run(cmd): |
96 | cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd | 132 | cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd |
97 | cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd | 133 | cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd |
98 | return runfetchcmd(cmd, d, workdir=workdir) | 134 | return runfetchcmd(cmd, d, workdir=workdir) |
99 | 135 | ||
100 | if self.configs: | ||
101 | for key, value in self.configs: | ||
102 | _run("npm config set %s %s" % (key, shlex.quote(value))) | ||
103 | |||
104 | if configs: | 136 | if configs: |
137 | bb.warn("Use of configs argument of NpmEnvironment.run() function" | ||
138 | " is deprecated. Please use args argument instead.") | ||
105 | for key, value in configs: | 139 | for key, value in configs: |
106 | _run("npm config set %s %s" % (key, shlex.quote(value))) | 140 | cmd += " --%s=%s" % (key, shlex.quote(value)) |
107 | 141 | ||
108 | if args: | 142 | if args: |
109 | for key, value in args: | 143 | for key, value in args: |
@@ -142,12 +176,12 @@ class Npm(FetchMethod): | |||
142 | raise ParameterError("Invalid 'version' parameter", ud.url) | 176 | raise ParameterError("Invalid 'version' parameter", ud.url) |
143 | 177 | ||
144 | # Extract the 'registry' part of the url | 178 | # Extract the 'registry' part of the url |
145 | ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) | 179 | ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) |
146 | 180 | ||
147 | # Using the 'downloadfilename' parameter as local filename | 181 | # Using the 'downloadfilename' parameter as local filename |
148 | # or the npm package name. | 182 | # or the npm package name. |
149 | if "downloadfilename" in ud.parm: | 183 | if "downloadfilename" in ud.parm: |
150 | ud.localfile = d.expand(ud.parm["downloadfilename"]) | 184 | ud.localfile = npm_localfile(ud.parm["downloadfilename"]) |
151 | else: | 185 | else: |
152 | ud.localfile = npm_localfile(ud.package, ud.version) | 186 | ud.localfile = npm_localfile(ud.package, ud.version) |
153 | 187 | ||
@@ -165,14 +199,14 @@ class Npm(FetchMethod): | |||
165 | 199 | ||
166 | def _resolve_proxy_url(self, ud, d): | 200 | def _resolve_proxy_url(self, ud, d): |
167 | def _npm_view(): | 201 | def _npm_view(): |
168 | configs = [] | 202 | args = [] |
169 | configs.append(("json", "true")) | 203 | args.append(("json", "true")) |
170 | configs.append(("registry", ud.registry)) | 204 | args.append(("registry", ud.registry)) |
171 | pkgver = shlex.quote(ud.package + "@" + ud.version) | 205 | pkgver = shlex.quote(ud.package + "@" + ud.version) |
172 | cmd = ud.basecmd + " view %s" % pkgver | 206 | cmd = ud.basecmd + " view %s" % pkgver |
173 | env = NpmEnvironment(d) | 207 | env = NpmEnvironment(d) |
174 | check_network_access(d, cmd, ud.registry) | 208 | check_network_access(d, cmd, ud.registry) |
175 | view_string = env.run(cmd, configs=configs) | 209 | view_string = env.run(cmd, args=args) |
176 | 210 | ||
177 | if not view_string: | 211 | if not view_string: |
178 | raise FetchError("Unavailable package %s" % pkgver, ud.url) | 212 | raise FetchError("Unavailable package %s" % pkgver, ud.url) |
@@ -280,6 +314,7 @@ class Npm(FetchMethod): | |||
280 | destsuffix = ud.parm.get("destsuffix", "npm") | 314 | destsuffix = ud.parm.get("destsuffix", "npm") |
281 | destdir = os.path.join(rootdir, destsuffix) | 315 | destdir = os.path.join(rootdir, destsuffix) |
282 | npm_unpack(ud.localpath, destdir, d) | 316 | npm_unpack(ud.localpath, destdir, d) |
317 | ud.unpack_tracer.unpack("npm", destdir) | ||
283 | 318 | ||
284 | def clean(self, ud, d): | 319 | def clean(self, ud, d): |
285 | """Clean any existing full or partial download""" | 320 | """Clean any existing full or partial download""" |
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py index 0c3511d8ab..2f9599ee9e 100644 --- a/bitbake/lib/bb/fetch2/npmsw.py +++ b/bitbake/lib/bb/fetch2/npmsw.py | |||
@@ -24,34 +24,39 @@ import bb | |||
24 | from bb.fetch2 import Fetch | 24 | from bb.fetch2 import Fetch |
25 | from bb.fetch2 import FetchMethod | 25 | from bb.fetch2 import FetchMethod |
26 | from bb.fetch2 import ParameterError | 26 | from bb.fetch2 import ParameterError |
27 | from bb.fetch2 import runfetchcmd | ||
27 | from bb.fetch2 import URI | 28 | from bb.fetch2 import URI |
28 | from bb.fetch2.npm import npm_integrity | 29 | from bb.fetch2.npm import npm_integrity |
29 | from bb.fetch2.npm import npm_localfile | 30 | from bb.fetch2.npm import npm_localfile |
30 | from bb.fetch2.npm import npm_unpack | 31 | from bb.fetch2.npm import npm_unpack |
31 | from bb.utils import is_semver | 32 | from bb.utils import is_semver |
33 | from bb.utils import lockfile | ||
34 | from bb.utils import unlockfile | ||
32 | 35 | ||
33 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): | 36 | def foreach_dependencies(shrinkwrap, callback=None, dev=False): |
34 | """ | 37 | """ |
35 | Run a callback for each dependencies of a shrinkwrap file. | 38 | Run a callback for each dependencies of a shrinkwrap file. |
36 | The callback is using the format: | 39 | The callback is using the format: |
37 | callback(name, params, deptree) | 40 | callback(name, data, location) |
38 | with: | 41 | with: |
39 | name = the package name (string) | 42 | name = the package name (string) |
40 | params = the package parameters (dictionary) | 43 | data = the package data (dictionary) |
41 | deptree = the package dependency tree (array of strings) | 44 | location = the location of the package (string) |
42 | """ | 45 | """ |
43 | def _walk_deps(deps, deptree): | 46 | packages = shrinkwrap.get("packages") |
44 | for name in deps: | 47 | if not packages: |
45 | subtree = [*deptree, name] | 48 | raise FetchError("Invalid shrinkwrap file format") |
46 | _walk_deps(deps[name].get("dependencies", {}), subtree) | 49 | |
47 | if callback is not None: | 50 | for location, data in packages.items(): |
48 | if deps[name].get("dev", False) and not dev: | 51 | # Skip empty main and local link target packages |
49 | continue | 52 | if not location.startswith('node_modules/'): |
50 | elif deps[name].get("bundled", False): | 53 | continue |
51 | continue | 54 | elif not dev and data.get("dev", False): |
52 | callback(name, deps[name], subtree) | 55 | continue |
53 | 56 | elif data.get("inBundle", False): | |
54 | _walk_deps(shrinkwrap.get("dependencies", {}), []) | 57 | continue |
58 | name = location.split('node_modules/')[-1] | ||
59 | callback(name, data, location) | ||
55 | 60 | ||
56 | class NpmShrinkWrap(FetchMethod): | 61 | class NpmShrinkWrap(FetchMethod): |
57 | """Class to fetch all package from a shrinkwrap file""" | 62 | """Class to fetch all package from a shrinkwrap file""" |
@@ -72,19 +77,28 @@ class NpmShrinkWrap(FetchMethod): | |||
72 | # Resolve the dependencies | 77 | # Resolve the dependencies |
73 | ud.deps = [] | 78 | ud.deps = [] |
74 | 79 | ||
75 | def _resolve_dependency(name, params, deptree): | 80 | def _resolve_dependency(name, params, destsuffix): |
76 | url = None | 81 | url = None |
77 | localpath = None | 82 | localpath = None |
78 | extrapaths = [] | 83 | extrapaths = [] |
79 | destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] | 84 | unpack = True |
80 | destsuffix = os.path.join(*destsubdirs) | ||
81 | 85 | ||
82 | integrity = params.get("integrity", None) | 86 | integrity = params.get("integrity") |
83 | resolved = params.get("resolved", None) | 87 | resolved = params.get("resolved") |
84 | version = params.get("version", None) | 88 | version = params.get("version") |
89 | link = params.get("link", False) | ||
90 | |||
91 | # Handle link sources | ||
92 | if link: | ||
93 | localpath = resolved | ||
94 | unpack = False | ||
85 | 95 | ||
86 | # Handle registry sources | 96 | # Handle registry sources |
87 | if is_semver(version) and resolved and integrity: | 97 | elif version and is_semver(version) and integrity: |
98 | # Handle duplicate dependencies without url | ||
99 | if not resolved: | ||
100 | return | ||
101 | |||
88 | localfile = npm_localfile(name, version) | 102 | localfile = npm_localfile(name, version) |
89 | 103 | ||
90 | uri = URI(resolved) | 104 | uri = URI(resolved) |
@@ -108,10 +122,10 @@ class NpmShrinkWrap(FetchMethod): | |||
108 | extrapaths.append(resolvefile) | 122 | extrapaths.append(resolvefile) |
109 | 123 | ||
110 | # Handle http tarball sources | 124 | # Handle http tarball sources |
111 | elif version.startswith("http") and integrity: | 125 | elif resolved.startswith("http") and integrity: |
112 | localfile = os.path.join("npm2", os.path.basename(version)) | 126 | localfile = npm_localfile(os.path.basename(resolved)) |
113 | 127 | ||
114 | uri = URI(version) | 128 | uri = URI(resolved) |
115 | uri.params["downloadfilename"] = localfile | 129 | uri.params["downloadfilename"] = localfile |
116 | 130 | ||
117 | checksum_name, checksum_expected = npm_integrity(integrity) | 131 | checksum_name, checksum_expected = npm_integrity(integrity) |
@@ -121,8 +135,12 @@ class NpmShrinkWrap(FetchMethod): | |||
121 | 135 | ||
122 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) | 136 | localpath = os.path.join(d.getVar("DL_DIR"), localfile) |
123 | 137 | ||
138 | # Handle local tarball sources | ||
139 | elif resolved.startswith("file"): | ||
140 | localpath = resolved[5:] | ||
141 | |||
124 | # Handle git sources | 142 | # Handle git sources |
125 | elif version.startswith("git"): | 143 | elif resolved.startswith("git"): |
126 | regex = re.compile(r""" | 144 | regex = re.compile(r""" |
127 | ^ | 145 | ^ |
128 | git\+ | 146 | git\+ |
@@ -134,29 +152,31 @@ class NpmShrinkWrap(FetchMethod): | |||
134 | $ | 152 | $ |
135 | """, re.VERBOSE) | 153 | """, re.VERBOSE) |
136 | 154 | ||
137 | match = regex.match(version) | 155 | match = regex.match(resolved) |
138 | |||
139 | if not match: | 156 | if not match: |
140 | raise ParameterError("Invalid git url: %s" % version, ud.url) | 157 | raise ParameterError("Invalid git url: %s" % resolved, ud.url) |
141 | 158 | ||
142 | groups = match.groupdict() | 159 | groups = match.groupdict() |
143 | 160 | ||
144 | uri = URI("git://" + str(groups["url"])) | 161 | uri = URI("git://" + str(groups["url"])) |
145 | uri.params["protocol"] = str(groups["protocol"]) | 162 | uri.params["protocol"] = str(groups["protocol"]) |
146 | uri.params["rev"] = str(groups["rev"]) | 163 | uri.params["rev"] = str(groups["rev"]) |
164 | uri.params["nobranch"] = "1" | ||
147 | uri.params["destsuffix"] = destsuffix | 165 | uri.params["destsuffix"] = destsuffix |
148 | 166 | ||
149 | url = str(uri) | 167 | url = str(uri) |
150 | 168 | ||
151 | # local tarball sources and local link sources are unsupported | ||
152 | else: | 169 | else: |
153 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) | 170 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) |
154 | 171 | ||
172 | # name is needed by unpack tracer for module mapping | ||
155 | ud.deps.append({ | 173 | ud.deps.append({ |
174 | "name": name, | ||
156 | "url": url, | 175 | "url": url, |
157 | "localpath": localpath, | 176 | "localpath": localpath, |
158 | "extrapaths": extrapaths, | 177 | "extrapaths": extrapaths, |
159 | "destsuffix": destsuffix, | 178 | "destsuffix": destsuffix, |
179 | "unpack": unpack, | ||
160 | }) | 180 | }) |
161 | 181 | ||
162 | try: | 182 | try: |
@@ -177,17 +197,23 @@ class NpmShrinkWrap(FetchMethod): | |||
177 | # This fetcher resolves multiple URIs from a shrinkwrap file and then | 197 | # This fetcher resolves multiple URIs from a shrinkwrap file and then |
178 | # forwards it to a proxy fetcher. The management of the donestamp file, | 198 | # forwards it to a proxy fetcher. The management of the donestamp file, |
179 | # the lockfile and the checksums are forwarded to the proxy fetcher. | 199 | # the lockfile and the checksums are forwarded to the proxy fetcher. |
180 | ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) | 200 | shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]] |
201 | if shrinkwrap_urls: | ||
202 | ud.proxy = Fetch(shrinkwrap_urls, data) | ||
181 | ud.needdonestamp = False | 203 | ud.needdonestamp = False |
182 | 204 | ||
183 | @staticmethod | 205 | @staticmethod |
184 | def _foreach_proxy_method(ud, handle): | 206 | def _foreach_proxy_method(ud, handle): |
185 | returns = [] | 207 | returns = [] |
186 | for proxy_url in ud.proxy.urls: | 208 | #Check if there are dependencies before try to fetch them |
187 | proxy_ud = ud.proxy.ud[proxy_url] | 209 | if len(ud.deps) > 0: |
188 | proxy_d = ud.proxy.d | 210 | for proxy_url in ud.proxy.urls: |
189 | proxy_ud.setup_localpath(proxy_d) | 211 | proxy_ud = ud.proxy.ud[proxy_url] |
190 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | 212 | proxy_d = ud.proxy.d |
213 | proxy_ud.setup_localpath(proxy_d) | ||
214 | lf = lockfile(proxy_ud.lockfile) | ||
215 | returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) | ||
216 | unlockfile(lf) | ||
191 | return returns | 217 | return returns |
192 | 218 | ||
193 | def verify_donestamp(self, ud, d): | 219 | def verify_donestamp(self, ud, d): |
@@ -220,10 +246,11 @@ class NpmShrinkWrap(FetchMethod): | |||
220 | 246 | ||
221 | def unpack(self, ud, rootdir, d): | 247 | def unpack(self, ud, rootdir, d): |
222 | """Unpack the downloaded dependencies""" | 248 | """Unpack the downloaded dependencies""" |
223 | destdir = d.getVar("S") | 249 | destdir = rootdir |
224 | destsuffix = ud.parm.get("destsuffix") | 250 | destsuffix = ud.parm.get("destsuffix") |
225 | if destsuffix: | 251 | if destsuffix: |
226 | destdir = os.path.join(rootdir, destsuffix) | 252 | destdir = os.path.join(rootdir, destsuffix) |
253 | ud.unpack_tracer.unpack("npm-shrinkwrap", destdir) | ||
227 | 254 | ||
228 | bb.utils.mkdirhier(destdir) | 255 | bb.utils.mkdirhier(destdir) |
229 | bb.utils.copyfile(ud.shrinkwrap_file, | 256 | bb.utils.copyfile(ud.shrinkwrap_file, |
@@ -237,7 +264,16 @@ class NpmShrinkWrap(FetchMethod): | |||
237 | 264 | ||
238 | for dep in manual: | 265 | for dep in manual: |
239 | depdestdir = os.path.join(destdir, dep["destsuffix"]) | 266 | depdestdir = os.path.join(destdir, dep["destsuffix"]) |
240 | npm_unpack(dep["localpath"], depdestdir, d) | 267 | if dep["url"]: |
268 | npm_unpack(dep["localpath"], depdestdir, d) | ||
269 | else: | ||
270 | depsrcdir= os.path.join(destdir, dep["localpath"]) | ||
271 | if dep["unpack"]: | ||
272 | npm_unpack(depsrcdir, depdestdir, d) | ||
273 | else: | ||
274 | bb.utils.mkdirhier(depdestdir) | ||
275 | cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) | ||
276 | runfetchcmd(cmd, d, workdir=depdestdir) | ||
241 | 277 | ||
242 | def clean(self, ud, d): | 278 | def clean(self, ud, d): |
243 | """Clean any existing full or partial download""" | 279 | """Clean any existing full or partial download""" |
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py index d9ce44390c..495ac8a30a 100644 --- a/bitbake/lib/bb/fetch2/osc.py +++ b/bitbake/lib/bb/fetch2/osc.py | |||
@@ -1,4 +1,6 @@ | |||
1 | # | 1 | # |
2 | # Copyright BitBake Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
3 | # | 5 | # |
4 | """ | 6 | """ |
@@ -9,6 +11,7 @@ Based on the svn "Fetch" implementation. | |||
9 | 11 | ||
10 | import logging | 12 | import logging |
11 | import os | 13 | import os |
14 | import re | ||
12 | import bb | 15 | import bb |
13 | from bb.fetch2 import FetchMethod | 16 | from bb.fetch2 import FetchMethod |
14 | from bb.fetch2 import FetchError | 17 | from bb.fetch2 import FetchError |
@@ -36,6 +39,7 @@ class Osc(FetchMethod): | |||
36 | # Create paths to osc checkouts | 39 | # Create paths to osc checkouts |
37 | oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc") | 40 | oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc") |
38 | relpath = self._strip_leading_slashes(ud.path) | 41 | relpath = self._strip_leading_slashes(ud.path) |
42 | ud.oscdir = oscdir | ||
39 | ud.pkgdir = os.path.join(oscdir, ud.host) | 43 | ud.pkgdir = os.path.join(oscdir, ud.host) |
40 | ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) | 44 | ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) |
41 | 45 | ||
@@ -43,13 +47,13 @@ class Osc(FetchMethod): | |||
43 | ud.revision = ud.parm['rev'] | 47 | ud.revision = ud.parm['rev'] |
44 | else: | 48 | else: |
45 | pv = d.getVar("PV", False) | 49 | pv = d.getVar("PV", False) |
46 | rev = bb.fetch2.srcrev_internal_helper(ud, d) | 50 | rev = bb.fetch2.srcrev_internal_helper(ud, d, '') |
47 | if rev: | 51 | if rev: |
48 | ud.revision = rev | 52 | ud.revision = rev |
49 | else: | 53 | else: |
50 | ud.revision = "" | 54 | ud.revision = "" |
51 | 55 | ||
52 | ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision)) | 56 | ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision)) |
53 | 57 | ||
54 | def _buildosccommand(self, ud, d, command): | 58 | def _buildosccommand(self, ud, d, command): |
55 | """ | 59 | """ |
@@ -59,26 +63,49 @@ class Osc(FetchMethod): | |||
59 | 63 | ||
60 | basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc" | 64 | basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc" |
61 | 65 | ||
62 | proto = ud.parm.get('protocol', 'ocs') | 66 | proto = ud.parm.get('protocol', 'https') |
63 | 67 | ||
64 | options = [] | 68 | options = [] |
65 | 69 | ||
66 | config = "-c %s" % self.generate_config(ud, d) | 70 | config = "-c %s" % self.generate_config(ud, d) |
67 | 71 | ||
68 | if ud.revision: | 72 | if getattr(ud, 'revision', ''): |
69 | options.append("-r %s" % ud.revision) | 73 | options.append("-r %s" % ud.revision) |
70 | 74 | ||
71 | coroot = self._strip_leading_slashes(ud.path) | 75 | coroot = self._strip_leading_slashes(ud.path) |
72 | 76 | ||
73 | if command == "fetch": | 77 | if command == "fetch": |
74 | osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) | 78 | osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options)) |
75 | elif command == "update": | 79 | elif command == "update": |
76 | osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) | 80 | osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options)) |
81 | elif command == "api_source": | ||
82 | osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module) | ||
77 | else: | 83 | else: |
78 | raise FetchError("Invalid osc command %s" % command, ud.url) | 84 | raise FetchError("Invalid osc command %s" % command, ud.url) |
79 | 85 | ||
80 | return osccmd | 86 | return osccmd |
81 | 87 | ||
88 | def _latest_revision(self, ud, d, name): | ||
89 | """ | ||
90 | Fetch latest revision for the given package | ||
91 | """ | ||
92 | api_source_cmd = self._buildosccommand(ud, d, "api_source") | ||
93 | |||
94 | output = runfetchcmd(api_source_cmd, d) | ||
95 | match = re.match(r'<directory ?.* rev="(\d+)".*>', output) | ||
96 | if match is None: | ||
97 | raise FetchError("Unable to parse osc response", ud.url) | ||
98 | return match.groups()[0] | ||
99 | |||
100 | def _revision_key(self, ud, d, name): | ||
101 | """ | ||
102 | Return a unique key for the url | ||
103 | """ | ||
104 | # Collapse adjacent slashes | ||
105 | slash_re = re.compile(r"/+") | ||
106 | rev = getattr(ud, 'revision', "latest") | ||
107 | return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev) | ||
108 | |||
82 | def download(self, ud, d): | 109 | def download(self, ud, d): |
83 | """ | 110 | """ |
84 | Fetch url | 111 | Fetch url |
@@ -86,7 +113,7 @@ class Osc(FetchMethod): | |||
86 | 113 | ||
87 | logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") | 114 | logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") |
88 | 115 | ||
89 | if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK): | 116 | if os.access(ud.moddir, os.R_OK): |
90 | oscupdatecmd = self._buildosccommand(ud, d, "update") | 117 | oscupdatecmd = self._buildosccommand(ud, d, "update") |
91 | logger.info("Update "+ ud.url) | 118 | logger.info("Update "+ ud.url) |
92 | # update sources there | 119 | # update sources there |
@@ -114,20 +141,23 @@ class Osc(FetchMethod): | |||
114 | Generate a .oscrc to be used for this run. | 141 | Generate a .oscrc to be used for this run. |
115 | """ | 142 | """ |
116 | 143 | ||
117 | config_path = os.path.join(d.getVar('OSCDIR'), "oscrc") | 144 | config_path = os.path.join(ud.oscdir, "oscrc") |
145 | if not os.path.exists(ud.oscdir): | ||
146 | bb.utils.mkdirhier(ud.oscdir) | ||
147 | |||
118 | if (os.path.exists(config_path)): | 148 | if (os.path.exists(config_path)): |
119 | os.remove(config_path) | 149 | os.remove(config_path) |
120 | 150 | ||
121 | f = open(config_path, 'w') | 151 | f = open(config_path, 'w') |
152 | proto = ud.parm.get('protocol', 'https') | ||
122 | f.write("[general]\n") | 153 | f.write("[general]\n") |
123 | f.write("apisrv = %s\n" % ud.host) | 154 | f.write("apiurl = %s://%s\n" % (proto, ud.host)) |
124 | f.write("scheme = http\n") | ||
125 | f.write("su-wrapper = su -c\n") | 155 | f.write("su-wrapper = su -c\n") |
126 | f.write("build-root = %s\n" % d.getVar('WORKDIR')) | 156 | f.write("build-root = %s\n" % d.getVar('WORKDIR')) |
127 | f.write("urllist = %s\n" % d.getVar("OSCURLLIST")) | 157 | f.write("urllist = %s\n" % d.getVar("OSCURLLIST")) |
128 | f.write("extra-pkgs = gzip\n") | 158 | f.write("extra-pkgs = gzip\n") |
129 | f.write("\n") | 159 | f.write("\n") |
130 | f.write("[%s]\n" % ud.host) | 160 | f.write("[%s://%s]\n" % (proto, ud.host)) |
131 | f.write("user = %s\n" % ud.parm["user"]) | 161 | f.write("user = %s\n" % ud.parm["user"]) |
132 | f.write("pass = %s\n" % ud.parm["pswd"]) | 162 | f.write("pass = %s\n" % ud.parm["pswd"]) |
133 | f.close() | 163 | f.close() |
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py index e2a41a4a12..3b6fa4b1ec 100644 --- a/bitbake/lib/bb/fetch2/perforce.py +++ b/bitbake/lib/bb/fetch2/perforce.py | |||
@@ -134,7 +134,7 @@ class Perforce(FetchMethod): | |||
134 | 134 | ||
135 | ud.setup_revisions(d) | 135 | ud.setup_revisions(d) |
136 | 136 | ||
137 | ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision)) | 137 | ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision)) |
138 | 138 | ||
139 | def _buildp4command(self, ud, d, command, depot_filename=None): | 139 | def _buildp4command(self, ud, d, command, depot_filename=None): |
140 | """ | 140 | """ |
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py index ffca73c8e4..22c0538139 100644 --- a/bitbake/lib/bb/fetch2/s3.py +++ b/bitbake/lib/bb/fetch2/s3.py | |||
@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use. | |||
18 | import os | 18 | import os |
19 | import bb | 19 | import bb |
20 | import urllib.request, urllib.parse, urllib.error | 20 | import urllib.request, urllib.parse, urllib.error |
21 | import re | ||
21 | from bb.fetch2 import FetchMethod | 22 | from bb.fetch2 import FetchMethod |
22 | from bb.fetch2 import FetchError | 23 | from bb.fetch2 import FetchError |
23 | from bb.fetch2 import runfetchcmd | 24 | from bb.fetch2 import runfetchcmd |
24 | 25 | ||
26 | def convertToBytes(value, unit): | ||
27 | value = float(value) | ||
28 | if (unit == "KiB"): | ||
29 | value = value*1024.0; | ||
30 | elif (unit == "MiB"): | ||
31 | value = value*1024.0*1024.0; | ||
32 | elif (unit == "GiB"): | ||
33 | value = value*1024.0*1024.0*1024.0; | ||
34 | return value | ||
35 | |||
36 | class S3ProgressHandler(bb.progress.LineFilterProgressHandler): | ||
37 | """ | ||
38 | Extract progress information from s3 cp output, e.g.: | ||
39 | Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining | ||
40 | """ | ||
41 | def __init__(self, d): | ||
42 | super(S3ProgressHandler, self).__init__(d) | ||
43 | # Send an initial progress event so the bar gets shown | ||
44 | self._fire_progress(0) | ||
45 | |||
46 | def writeline(self, line): | ||
47 | percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line) | ||
48 | if percs: | ||
49 | completed = (percs[-1][0]) | ||
50 | completedUnit = (percs[-1][1]) | ||
51 | total = (percs[-1][2]) | ||
52 | totalUnit = (percs[-1][3]) | ||
53 | completed = convertToBytes(completed, completedUnit) | ||
54 | total = convertToBytes(total, totalUnit) | ||
55 | progress = (completed/total)*100.0 | ||
56 | rate = percs[-1][4] | ||
57 | self.update(progress, rate) | ||
58 | return False | ||
59 | return True | ||
60 | |||
61 | |||
25 | class S3(FetchMethod): | 62 | class S3(FetchMethod): |
26 | """Class to fetch urls via 'aws s3'""" | 63 | """Class to fetch urls via 'aws s3'""" |
27 | 64 | ||
@@ -40,7 +77,7 @@ class S3(FetchMethod): | |||
40 | else: | 77 | else: |
41 | ud.basename = os.path.basename(ud.path) | 78 | ud.basename = os.path.basename(ud.path) |
42 | 79 | ||
43 | ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) | 80 | ud.localfile = ud.basename |
44 | 81 | ||
45 | ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3" | 82 | ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3" |
46 | 83 | ||
@@ -52,7 +89,9 @@ class S3(FetchMethod): | |||
52 | 89 | ||
53 | cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) | 90 | cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) |
54 | bb.fetch2.check_network_access(d, cmd, ud.url) | 91 | bb.fetch2.check_network_access(d, cmd, ud.url) |
55 | runfetchcmd(cmd, d) | 92 | |
93 | progresshandler = S3ProgressHandler(d) | ||
94 | runfetchcmd(cmd, d, False, log=progresshandler) | ||
56 | 95 | ||
57 | # Additional sanity checks copied from the wget class (although there | 96 | # Additional sanity checks copied from the wget class (although there |
58 | # are no known issues which mean these are required, treat the aws cli | 97 | # are no known issues which mean these are required, treat the aws cli |
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py index f87f292e5d..bee71a0d0d 100644 --- a/bitbake/lib/bb/fetch2/sftp.py +++ b/bitbake/lib/bb/fetch2/sftp.py | |||
@@ -77,7 +77,7 @@ class SFTP(FetchMethod): | |||
77 | else: | 77 | else: |
78 | ud.basename = os.path.basename(ud.path) | 78 | ud.basename = os.path.basename(ud.path) |
79 | 79 | ||
80 | ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) | 80 | ud.localfile = ud.basename |
81 | 81 | ||
82 | def download(self, ud, d): | 82 | def download(self, ud, d): |
83 | """Fetch urls""" | 83 | """Fetch urls""" |
@@ -103,7 +103,7 @@ class SFTP(FetchMethod): | |||
103 | if path[:3] == '/~/': | 103 | if path[:3] == '/~/': |
104 | path = path[3:] | 104 | path = path[3:] |
105 | 105 | ||
106 | remote = '%s%s:%s' % (user, urlo.hostname, path) | 106 | remote = '"%s%s:%s"' % (user, urlo.hostname, path) |
107 | 107 | ||
108 | cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) | 108 | cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) |
109 | 109 | ||
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py index 2c8557e1f8..2a0f2cb44b 100644 --- a/bitbake/lib/bb/fetch2/ssh.py +++ b/bitbake/lib/bb/fetch2/ssh.py | |||
@@ -32,6 +32,7 @@ IETF secsh internet draft: | |||
32 | 32 | ||
33 | import re, os | 33 | import re, os |
34 | from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd | 34 | from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd |
35 | import urllib | ||
35 | 36 | ||
36 | 37 | ||
37 | __pattern__ = re.compile(r''' | 38 | __pattern__ = re.compile(r''' |
@@ -40,9 +41,9 @@ __pattern__ = re.compile(r''' | |||
40 | ( # Optional username/password block | 41 | ( # Optional username/password block |
41 | (?P<user>\S+) # username | 42 | (?P<user>\S+) # username |
42 | (:(?P<pass>\S+))? # colon followed by the password (optional) | 43 | (:(?P<pass>\S+))? # colon followed by the password (optional) |
43 | )? | ||
44 | (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) | 44 | (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) |
45 | @ | 45 | @ |
46 | )? | ||
46 | (?P<host>\S+?) # non-greedy match of the host | 47 | (?P<host>\S+?) # non-greedy match of the host |
47 | (:(?P<port>[0-9]+))? # colon followed by the port (optional) | 48 | (:(?P<port>[0-9]+))? # colon followed by the port (optional) |
48 | / | 49 | / |
@@ -70,9 +71,9 @@ class SSH(FetchMethod): | |||
70 | "git:// prefix with protocol=ssh", urldata.url) | 71 | "git:// prefix with protocol=ssh", urldata.url) |
71 | m = __pattern__.match(urldata.url) | 72 | m = __pattern__.match(urldata.url) |
72 | path = m.group('path') | 73 | path = m.group('path') |
74 | path = urllib.parse.unquote(path) | ||
73 | host = m.group('host') | 75 | host = m.group('host') |
74 | urldata.localpath = os.path.join(d.getVar('DL_DIR'), | 76 | urldata.localfile = os.path.basename(os.path.normpath(path)) |
75 | os.path.basename(os.path.normpath(path))) | ||
76 | 77 | ||
77 | def download(self, urldata, d): | 78 | def download(self, urldata, d): |
78 | dldir = d.getVar('DL_DIR') | 79 | dldir = d.getVar('DL_DIR') |
@@ -96,6 +97,11 @@ class SSH(FetchMethod): | |||
96 | fr += '@%s' % host | 97 | fr += '@%s' % host |
97 | else: | 98 | else: |
98 | fr = host | 99 | fr = host |
100 | |||
101 | if path[0] != '~': | ||
102 | path = '/%s' % path | ||
103 | path = urllib.parse.unquote(path) | ||
104 | |||
99 | fr += ':%s' % path | 105 | fr += ':%s' % path |
100 | 106 | ||
101 | cmd = 'scp -B -r %s %s %s/' % ( | 107 | cmd = 'scp -B -r %s %s %s/' % ( |
@@ -108,3 +114,41 @@ class SSH(FetchMethod): | |||
108 | 114 | ||
109 | runfetchcmd(cmd, d) | 115 | runfetchcmd(cmd, d) |
110 | 116 | ||
117 | def checkstatus(self, fetch, urldata, d): | ||
118 | """ | ||
119 | Check the status of the url | ||
120 | """ | ||
121 | m = __pattern__.match(urldata.url) | ||
122 | path = m.group('path') | ||
123 | host = m.group('host') | ||
124 | port = m.group('port') | ||
125 | user = m.group('user') | ||
126 | password = m.group('pass') | ||
127 | |||
128 | if port: | ||
129 | portarg = '-P %s' % port | ||
130 | else: | ||
131 | portarg = '' | ||
132 | |||
133 | if user: | ||
134 | fr = user | ||
135 | if password: | ||
136 | fr += ':%s' % password | ||
137 | fr += '@%s' % host | ||
138 | else: | ||
139 | fr = host | ||
140 | |||
141 | if path[0] != '~': | ||
142 | path = '/%s' % path | ||
143 | path = urllib.parse.unquote(path) | ||
144 | |||
145 | cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % ( | ||
146 | portarg, | ||
147 | fr, | ||
148 | path | ||
149 | ) | ||
150 | |||
151 | check_network_access(d, cmd, urldata.url) | ||
152 | runfetchcmd(cmd, d) | ||
153 | |||
154 | return True | ||
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py index 8856ef1c62..0852108e7d 100644 --- a/bitbake/lib/bb/fetch2/svn.py +++ b/bitbake/lib/bb/fetch2/svn.py | |||
@@ -57,7 +57,12 @@ class Svn(FetchMethod): | |||
57 | if 'rev' in ud.parm: | 57 | if 'rev' in ud.parm: |
58 | ud.revision = ud.parm['rev'] | 58 | ud.revision = ud.parm['rev'] |
59 | 59 | ||
60 | ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision)) | 60 | # Whether to use the @REV peg-revision syntax in the svn command or not |
61 | ud.pegrevision = True | ||
62 | if 'nopegrevision' in ud.parm: | ||
63 | ud.pegrevision = False | ||
64 | |||
65 | ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision])) | ||
61 | 66 | ||
62 | def _buildsvncommand(self, ud, d, command): | 67 | def _buildsvncommand(self, ud, d, command): |
63 | """ | 68 | """ |
@@ -86,7 +91,7 @@ class Svn(FetchMethod): | |||
86 | if command == "info": | 91 | if command == "info": |
87 | svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) | 92 | svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) |
88 | elif command == "log1": | 93 | elif command == "log1": |
89 | svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) | 94 | svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) |
90 | else: | 95 | else: |
91 | suffix = "" | 96 | suffix = "" |
92 | 97 | ||
@@ -98,7 +103,8 @@ class Svn(FetchMethod): | |||
98 | 103 | ||
99 | if ud.revision: | 104 | if ud.revision: |
100 | options.append("-r %s" % ud.revision) | 105 | options.append("-r %s" % ud.revision) |
101 | suffix = "@%s" % (ud.revision) | 106 | if ud.pegrevision: |
107 | suffix = "@%s" % (ud.revision) | ||
102 | 108 | ||
103 | if command == "fetch": | 109 | if command == "fetch": |
104 | transportuser = ud.parm.get("transportuser", "") | 110 | transportuser = ud.parm.get("transportuser", "") |
@@ -204,3 +210,6 @@ class Svn(FetchMethod): | |||
204 | 210 | ||
205 | def _build_revision(self, ud, d): | 211 | def _build_revision(self, ud, d): |
206 | return ud.revision | 212 | return ud.revision |
213 | |||
214 | def supports_checksum(self, urldata): | ||
215 | return False | ||
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index 6d82f3af07..7e43d3bc97 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py | |||
@@ -26,7 +26,6 @@ from bb.fetch2 import FetchMethod | |||
26 | from bb.fetch2 import FetchError | 26 | from bb.fetch2 import FetchError |
27 | from bb.fetch2 import logger | 27 | from bb.fetch2 import logger |
28 | from bb.fetch2 import runfetchcmd | 28 | from bb.fetch2 import runfetchcmd |
29 | from bb.utils import export_proxies | ||
30 | from bs4 import BeautifulSoup | 29 | from bs4 import BeautifulSoup |
31 | from bs4 import SoupStrainer | 30 | from bs4 import SoupStrainer |
32 | 31 | ||
@@ -52,18 +51,19 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler): | |||
52 | 51 | ||
53 | 52 | ||
54 | class Wget(FetchMethod): | 53 | class Wget(FetchMethod): |
54 | """Class to fetch urls via 'wget'""" | ||
55 | 55 | ||
56 | # CDNs like CloudFlare may do a 'browser integrity test' which can fail | 56 | def check_certs(self, d): |
57 | # with the standard wget/urllib User-Agent, so pretend to be a modern | 57 | """ |
58 | # browser. | 58 | Should certificates be checked? |
59 | user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" | 59 | """ |
60 | return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0" | ||
60 | 61 | ||
61 | """Class to fetch urls via 'wget'""" | ||
62 | def supports(self, ud, d): | 62 | def supports(self, ud, d): |
63 | """ | 63 | """ |
64 | Check to see if a given url can be fetched with wget. | 64 | Check to see if a given url can be fetched with wget. |
65 | """ | 65 | """ |
66 | return ud.type in ['http', 'https', 'ftp'] | 66 | return ud.type in ['http', 'https', 'ftp', 'ftps'] |
67 | 67 | ||
68 | def recommends_checksum(self, urldata): | 68 | def recommends_checksum(self, urldata): |
69 | return True | 69 | return True |
@@ -78,11 +78,17 @@ class Wget(FetchMethod): | |||
78 | else: | 78 | else: |
79 | ud.basename = os.path.basename(ud.path) | 79 | ud.basename = os.path.basename(ud.path) |
80 | 80 | ||
81 | ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) | 81 | ud.localfile = ud.basename |
82 | if not ud.localfile: | 82 | if not ud.localfile: |
83 | ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) | 83 | ud.localfile = ud.host + ud.path.replace("/", ".") |
84 | 84 | ||
85 | self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" | 85 | self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100" |
86 | |||
87 | if ud.type == 'ftp' or ud.type == 'ftps': | ||
88 | self.basecmd += " --passive-ftp" | ||
89 | |||
90 | if not self.check_certs(d): | ||
91 | self.basecmd += " --no-check-certificate" | ||
86 | 92 | ||
87 | def _runwget(self, ud, d, command, quiet, workdir=None): | 93 | def _runwget(self, ud, d, command, quiet, workdir=None): |
88 | 94 | ||
@@ -90,39 +96,53 @@ class Wget(FetchMethod): | |||
90 | 96 | ||
91 | logger.debug2("Fetching %s using command '%s'" % (ud.url, command)) | 97 | logger.debug2("Fetching %s using command '%s'" % (ud.url, command)) |
92 | bb.fetch2.check_network_access(d, command, ud.url) | 98 | bb.fetch2.check_network_access(d, command, ud.url) |
93 | runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir) | 99 | runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir) |
94 | 100 | ||
95 | def download(self, ud, d): | 101 | def download(self, ud, d): |
96 | """Fetch urls""" | 102 | """Fetch urls""" |
97 | 103 | ||
98 | fetchcmd = self.basecmd | 104 | fetchcmd = self.basecmd |
99 | 105 | ||
100 | if 'downloadfilename' in ud.parm: | 106 | dldir = os.path.realpath(d.getVar("DL_DIR")) |
101 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | 107 | localpath = os.path.join(dldir, ud.localfile) + ".tmp" |
102 | bb.utils.mkdirhier(os.path.dirname(localpath)) | 108 | bb.utils.mkdirhier(os.path.dirname(localpath)) |
103 | fetchcmd += " -O %s" % shlex.quote(localpath) | 109 | fetchcmd += " --output-document=%s" % shlex.quote(localpath) |
104 | 110 | ||
105 | if ud.user and ud.pswd: | 111 | if ud.user and ud.pswd: |
106 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | 112 | fetchcmd += " --auth-no-challenge" |
113 | if ud.parm.get("redirectauth", "1") == "1": | ||
114 | # An undocumented feature of wget is that if the | ||
115 | # username/password are specified on the URI, wget will only | ||
116 | # send the Authorization header to the first host and not to | ||
117 | # any hosts that it is redirected to. With the increasing | ||
118 | # usage of temporary AWS URLs, this difference now matters as | ||
119 | # AWS will reject any request that has authentication both in | ||
120 | # the query parameters (from the redirect) and in the | ||
121 | # Authorization header. | ||
122 | fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd) | ||
107 | 123 | ||
108 | uri = ud.url.split(";")[0] | 124 | uri = ud.url.split(";")[0] |
109 | if os.path.exists(ud.localpath): | 125 | fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri) |
110 | # file exists, but we didnt complete it.. trying again.. | ||
111 | fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri) | ||
112 | else: | ||
113 | fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri) | ||
114 | |||
115 | self._runwget(ud, d, fetchcmd, False) | 126 | self._runwget(ud, d, fetchcmd, False) |
116 | 127 | ||
117 | # Sanity check since wget can pretend it succeed when it didn't | 128 | # Sanity check since wget can pretend it succeed when it didn't |
118 | # Also, this used to happen if sourceforge sent us to the mirror page | 129 | # Also, this used to happen if sourceforge sent us to the mirror page |
119 | if not os.path.exists(ud.localpath): | 130 | if not os.path.exists(localpath): |
120 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) | 131 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri) |
121 | 132 | ||
122 | if os.path.getsize(ud.localpath) == 0: | 133 | if os.path.getsize(localpath) == 0: |
123 | os.remove(ud.localpath) | 134 | os.remove(localpath) |
124 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) | 135 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) |
125 | 136 | ||
137 | # Try and verify any checksum now, meaning if it isn't correct, we don't remove the | ||
138 | # original file, which might be a race (imagine two recipes referencing the same | ||
139 | # source, one with an incorrect checksum) | ||
140 | bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False) | ||
141 | |||
142 | # Remove the ".tmp" and move the file into position atomically | ||
143 | # Our lock prevents multiple writers but mirroring code may grab incomplete files | ||
144 | os.rename(localpath, localpath[:-4]) | ||
145 | |||
126 | return True | 146 | return True |
127 | 147 | ||
128 | def checkstatus(self, fetch, ud, d, try_again=True): | 148 | def checkstatus(self, fetch, ud, d, try_again=True): |
@@ -209,12 +229,17 @@ class Wget(FetchMethod): | |||
209 | # We let the request fail and expect it to be | 229 | # We let the request fail and expect it to be |
210 | # tried once more ("try_again" in check_status()), | 230 | # tried once more ("try_again" in check_status()), |
211 | # with the dead connection removed from the cache. | 231 | # with the dead connection removed from the cache. |
212 | # If it still fails, we give up, which can happend for bad | 232 | # If it still fails, we give up, which can happen for bad |
213 | # HTTP proxy settings. | 233 | # HTTP proxy settings. |
214 | fetch.connection_cache.remove_connection(h.host, h.port) | 234 | fetch.connection_cache.remove_connection(h.host, h.port) |
215 | raise urllib.error.URLError(err) | 235 | raise urllib.error.URLError(err) |
216 | else: | 236 | else: |
217 | r = h.getresponse() | 237 | try: |
238 | r = h.getresponse() | ||
239 | except TimeoutError as e: | ||
240 | if fetch.connection_cache: | ||
241 | fetch.connection_cache.remove_connection(h.host, h.port) | ||
242 | raise TimeoutError(e) | ||
218 | 243 | ||
219 | # Pick apart the HTTPResponse object to get the addinfourl | 244 | # Pick apart the HTTPResponse object to get the addinfourl |
220 | # object initialized properly. | 245 | # object initialized properly. |
@@ -275,71 +300,115 @@ class Wget(FetchMethod): | |||
275 | 300 | ||
276 | class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): | 301 | class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): |
277 | """ | 302 | """ |
278 | urllib2.HTTPRedirectHandler resets the method to GET on redirect, | 303 | urllib2.HTTPRedirectHandler before 3.13 has two flaws: |
279 | when we want to follow redirects using the original method. | 304 | |
305 | It resets the method to GET on redirect when we want to follow | ||
306 | redirects using the original method (typically HEAD). This was fixed | ||
307 | in 759e8e7. | ||
308 | |||
309 | It also doesn't handle 308 (Permanent Redirect). This was fixed in | ||
310 | c379bc5. | ||
311 | |||
312 | Until we depend on Python 3.13 onwards, copy the redirect_request | ||
313 | method to fix these issues. | ||
280 | """ | 314 | """ |
281 | def redirect_request(self, req, fp, code, msg, headers, newurl): | 315 | def redirect_request(self, req, fp, code, msg, headers, newurl): |
282 | newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) | 316 | m = req.get_method() |
283 | newreq.get_method = req.get_method | 317 | if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD") |
284 | return newreq | 318 | or code in (301, 302, 303) and m == "POST")): |
285 | exported_proxies = export_proxies(d) | 319 | raise urllib.HTTPError(req.full_url, code, msg, headers, fp) |
286 | 320 | ||
287 | handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] | 321 | # Strictly (according to RFC 2616), 301 or 302 in response to |
288 | if exported_proxies: | 322 | # a POST MUST NOT cause a redirection without confirmation |
289 | handlers.append(urllib.request.ProxyHandler()) | 323 | # from the user (of urllib.request, in this case). In practice, |
290 | handlers.append(CacheHTTPHandler()) | 324 | # essentially all clients do redirect in this case, so we do |
291 | # Since Python 2.7.9 ssl cert validation is enabled by default | 325 | # the same. |
292 | # see PEP-0476, this causes verification errors on some https servers | 326 | |
293 | # so disable by default. | 327 | # Be conciliant with URIs containing a space. This is mainly |
294 | import ssl | 328 | # redundant with the more complete encoding done in http_error_302(), |
295 | if hasattr(ssl, '_create_unverified_context'): | 329 | # but it is kept for compatibility with other callers. |
296 | handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) | 330 | newurl = newurl.replace(' ', '%20') |
297 | opener = urllib.request.build_opener(*handlers) | 331 | |
298 | 332 | CONTENT_HEADERS = ("content-length", "content-type") | |
299 | try: | 333 | newheaders = {k: v for k, v in req.headers.items() |
300 | uri = ud.url.split(";")[0] | 334 | if k.lower() not in CONTENT_HEADERS} |
301 | r = urllib.request.Request(uri) | 335 | return urllib.request.Request(newurl, |
302 | r.get_method = lambda: "HEAD" | 336 | method="HEAD" if m == "HEAD" else "GET", |
303 | # Some servers (FusionForge, as used on Alioth) require that the | 337 | headers=newheaders, |
304 | # optional Accept header is set. | 338 | origin_req_host=req.origin_req_host, |
305 | r.add_header("Accept", "*/*") | 339 | unverifiable=True) |
306 | r.add_header("User-Agent", self.user_agent) | 340 | |
307 | def add_basic_auth(login_str, request): | 341 | http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302 |
308 | '''Adds Basic auth to http request, pass in login:password as string''' | 342 | |
309 | import base64 | 343 | # We need to update the environment here as both the proxy and HTTPS |
310 | encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") | 344 | # handlers need variables set. The proxy needs http_proxy and friends to |
311 | authheader = "Basic %s" % encodeuser | 345 | # be set, and HTTPSHandler ends up calling into openssl to load the |
312 | r.add_header("Authorization", authheader) | 346 | # certificates. In buildtools configurations this will be looking at the |
313 | 347 | # wrong place for certificates by default: we set SSL_CERT_FILE to the | |
314 | if ud.user and ud.pswd: | 348 | # right location in the buildtools environment script but as BitBake |
315 | add_basic_auth(ud.user + ':' + ud.pswd, r) | 349 | # prunes prunes the environment this is lost. When binaries are executed |
350 | # runfetchcmd ensures these values are in the environment, but this is | ||
351 | # pure Python so we need to update the environment. | ||
352 | # | ||
353 | # Avoid tramping the environment too much by using bb.utils.environment | ||
354 | # to scope the changes to the build_opener request, which is when the | ||
355 | # environment lookups happen. | ||
356 | newenv = bb.fetch2.get_fetcher_environment(d) | ||
357 | |||
358 | with bb.utils.environment(**newenv): | ||
359 | import ssl | ||
360 | |||
361 | if self.check_certs(d): | ||
362 | context = ssl.create_default_context() | ||
363 | else: | ||
364 | context = ssl._create_unverified_context() | ||
365 | |||
366 | handlers = [FixedHTTPRedirectHandler, | ||
367 | HTTPMethodFallback, | ||
368 | urllib.request.ProxyHandler(), | ||
369 | CacheHTTPHandler(), | ||
370 | urllib.request.HTTPSHandler(context=context)] | ||
371 | opener = urllib.request.build_opener(*handlers) | ||
316 | 372 | ||
317 | try: | 373 | try: |
318 | import netrc | 374 | parts = urllib.parse.urlparse(ud.url.split(";")[0]) |
319 | n = netrc.netrc() | 375 | uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path) |
320 | login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) | 376 | r = urllib.request.Request(uri) |
321 | add_basic_auth("%s:%s" % (login, password), r) | 377 | r.get_method = lambda: "HEAD" |
322 | except (TypeError, ImportError, IOError, netrc.NetrcParseError): | 378 | # Some servers (FusionForge, as used on Alioth) require that the |
323 | pass | 379 | # optional Accept header is set. |
324 | 380 | r.add_header("Accept", "*/*") | |
325 | with opener.open(r) as response: | 381 | r.add_header("User-Agent", "bitbake/{}".format(bb.__version__)) |
326 | pass | 382 | def add_basic_auth(login_str, request): |
327 | except urllib.error.URLError as e: | 383 | '''Adds Basic auth to http request, pass in login:password as string''' |
328 | if try_again: | 384 | import base64 |
329 | logger.debug2("checkstatus: trying again") | 385 | encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") |
330 | return self.checkstatus(fetch, ud, d, False) | 386 | authheader = "Basic %s" % encodeuser |
331 | else: | 387 | r.add_header("Authorization", authheader) |
332 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | 388 | |
333 | logger.debug2("checkstatus() urlopen failed: %s" % e) | 389 | if ud.user and ud.pswd: |
334 | return False | 390 | add_basic_auth(ud.user + ':' + ud.pswd, r) |
335 | except ConnectionResetError as e: | 391 | |
336 | if try_again: | 392 | try: |
337 | logger.debug2("checkstatus: trying again") | 393 | import netrc |
338 | return self.checkstatus(fetch, ud, d, False) | 394 | auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname) |
339 | else: | 395 | if auth_data: |
340 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | 396 | login, _, password = auth_data |
341 | logger.debug2("checkstatus() urlopen failed: %s" % e) | 397 | add_basic_auth("%s:%s" % (login, password), r) |
342 | return False | 398 | except (FileNotFoundError, netrc.NetrcParseError): |
399 | pass | ||
400 | |||
401 | with opener.open(r, timeout=100) as response: | ||
402 | pass | ||
403 | except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e: | ||
404 | if try_again: | ||
405 | logger.debug2("checkstatus: trying again") | ||
406 | return self.checkstatus(fetch, ud, d, False) | ||
407 | else: | ||
408 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | ||
409 | logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e)) | ||
410 | return False | ||
411 | |||
343 | return True | 412 | return True |
344 | 413 | ||
345 | def _parse_path(self, regex, s): | 414 | def _parse_path(self, regex, s): |
@@ -416,7 +485,7 @@ class Wget(FetchMethod): | |||
416 | f = tempfile.NamedTemporaryFile() | 485 | f = tempfile.NamedTemporaryFile() |
417 | with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: | 486 | with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: |
418 | fetchcmd = self.basecmd | 487 | fetchcmd = self.basecmd |
419 | fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'" | 488 | fetchcmd += " --output-document=%s '%s'" % (f.name, uri) |
420 | try: | 489 | try: |
421 | self._runwget(ud, d, fetchcmd, True, workdir=workdir) | 490 | self._runwget(ud, d, fetchcmd, True, workdir=workdir) |
422 | fetchresult = f.read() | 491 | fetchresult = f.read() |
@@ -472,7 +541,7 @@ class Wget(FetchMethod): | |||
472 | version_dir = ['', '', ''] | 541 | version_dir = ['', '', ''] |
473 | version = ['', '', ''] | 542 | version = ['', '', ''] |
474 | 543 | ||
475 | dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") | 544 | dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))") |
476 | s = dirver_regex.search(dirver) | 545 | s = dirver_regex.search(dirver) |
477 | if s: | 546 | if s: |
478 | version_dir[1] = s.group('ver') | 547 | version_dir[1] = s.group('ver') |
@@ -548,7 +617,7 @@ class Wget(FetchMethod): | |||
548 | 617 | ||
549 | # src.rpm extension was added only for rpm package. Can be removed if the rpm | 618 | # src.rpm extension was added only for rpm package. Can be removed if the rpm |
550 | # packaged will always be considered as having to be manually upgraded | 619 | # packaged will always be considered as having to be manually upgraded |
551 | psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" | 620 | psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)" |
552 | 621 | ||
553 | # match name, version and archive type of a package | 622 | # match name, version and archive type of a package |
554 | package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" | 623 | package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" |
@@ -576,13 +645,17 @@ class Wget(FetchMethod): | |||
576 | 645 | ||
577 | sanity check to ensure same name and type. | 646 | sanity check to ensure same name and type. |
578 | """ | 647 | """ |
579 | package = ud.path.split("/")[-1] | 648 | if 'downloadfilename' in ud.parm: |
649 | package = ud.parm['downloadfilename'] | ||
650 | else: | ||
651 | package = ud.path.split("/")[-1] | ||
580 | current_version = ['', d.getVar('PV'), ''] | 652 | current_version = ['', d.getVar('PV'), ''] |
581 | 653 | ||
582 | """possible to have no version in pkg name, such as spectrum-fw""" | 654 | """possible to have no version in pkg name, such as spectrum-fw""" |
583 | if not re.search(r"\d+", package): | 655 | if not re.search(r"\d+", package): |
584 | current_version[1] = re.sub('_', '.', current_version[1]) | 656 | current_version[1] = re.sub('_', '.', current_version[1]) |
585 | current_version[1] = re.sub('-', '.', current_version[1]) | 657 | current_version[1] = re.sub('-', '.', current_version[1]) |
658 | bb.debug(3, "latest_versionstring: no version found in %s" % package) | ||
586 | return (current_version[1], '') | 659 | return (current_version[1], '') |
587 | 660 | ||
588 | package_regex = self._init_regexes(package, ud, d) | 661 | package_regex = self._init_regexes(package, ud, d) |
@@ -599,10 +672,10 @@ class Wget(FetchMethod): | |||
599 | # search for version matches on folders inside the path, like: | 672 | # search for version matches on folders inside the path, like: |
600 | # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz | 673 | # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz |
601 | dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") | 674 | dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") |
602 | m = dirver_regex.search(path) | 675 | m = dirver_regex.findall(path) |
603 | if m: | 676 | if m: |
604 | pn = d.getVar('PN') | 677 | pn = d.getVar('PN') |
605 | dirver = m.group('dirver') | 678 | dirver = m[-1][0] |
606 | 679 | ||
607 | dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) | 680 | dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) |
608 | if not dirver_pn_regex.search(dirver): | 681 | if not dirver_pn_regex.search(dirver): |