diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 294 |
1 files changed, 229 insertions, 65 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index e3ba80a3f5..c7ff769fdf 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -44,13 +44,27 @@ Supported SRC_URI options are: | |||
44 | 44 | ||
45 | - nobranch | 45 | - nobranch |
46 | Don't check the SHA validation for branch. set this option for the recipe | 46 | Don't check the SHA validation for branch. set this option for the recipe |
47 | referring to commit which is valid in tag instead of branch. | 47 | referring to commit which is valid in any namespace (branch, tag, ...) |
48 | instead of branch. | ||
48 | The default is "0", set nobranch=1 if needed. | 49 | The default is "0", set nobranch=1 if needed. |
49 | 50 | ||
51 | - subpath | ||
52 | Limit the checkout to a specific subpath of the tree. | ||
53 | By default, checkout the whole tree, set subpath=<path> if needed | ||
54 | |||
55 | - destsuffix | ||
56 | The name of the path in which to place the checkout. | ||
57 | By default, the path is git/, set destsuffix=<suffix> if needed | ||
58 | |||
50 | - usehead | 59 | - usehead |
51 | For local git:// urls to use the current branch HEAD as the revision for use with | 60 | For local git:// urls to use the current branch HEAD as the revision for use with |
52 | AUTOREV. Implies nobranch. | 61 | AUTOREV. Implies nobranch. |
53 | 62 | ||
63 | - lfs | ||
64 | Enable the checkout to use LFS for large files. This will download all LFS files | ||
65 | in the download step, as the unpack step does not have network access. | ||
66 | The default is "1", set lfs=0 to skip. | ||
67 | |||
54 | """ | 68 | """ |
55 | 69 | ||
56 | # Copyright (C) 2005 Richard Purdie | 70 | # Copyright (C) 2005 Richard Purdie |
@@ -64,14 +78,20 @@ import fnmatch | |||
64 | import os | 78 | import os |
65 | import re | 79 | import re |
66 | import shlex | 80 | import shlex |
81 | import shutil | ||
67 | import subprocess | 82 | import subprocess |
68 | import tempfile | 83 | import tempfile |
69 | import bb | 84 | import bb |
70 | import bb.progress | 85 | import bb.progress |
86 | from contextlib import contextmanager | ||
71 | from bb.fetch2 import FetchMethod | 87 | from bb.fetch2 import FetchMethod |
72 | from bb.fetch2 import runfetchcmd | 88 | from bb.fetch2 import runfetchcmd |
73 | from bb.fetch2 import logger | 89 | from bb.fetch2 import logger |
90 | from bb.fetch2 import trusted_network | ||
91 | |||
74 | 92 | ||
93 | sha1_re = re.compile(r'^[0-9a-f]{40}$') | ||
94 | slash_re = re.compile(r"/+") | ||
75 | 95 | ||
76 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): | 96 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): |
77 | """Extract progress information from git output""" | 97 | """Extract progress information from git output""" |
@@ -130,6 +150,9 @@ class Git(FetchMethod): | |||
130 | def supports_checksum(self, urldata): | 150 | def supports_checksum(self, urldata): |
131 | return False | 151 | return False |
132 | 152 | ||
153 | def cleanup_upon_failure(self): | ||
154 | return False | ||
155 | |||
133 | def urldata_init(self, ud, d): | 156 | def urldata_init(self, ud, d): |
134 | """ | 157 | """ |
135 | init git specific variable within url data | 158 | init git specific variable within url data |
@@ -141,6 +164,11 @@ class Git(FetchMethod): | |||
141 | ud.proto = 'file' | 164 | ud.proto = 'file' |
142 | else: | 165 | else: |
143 | ud.proto = "git" | 166 | ud.proto = "git" |
167 | if ud.host == "github.com" and ud.proto == "git": | ||
168 | # github stopped supporting git protocol | ||
169 | # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git | ||
170 | ud.proto = "https" | ||
171 | bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url) | ||
144 | 172 | ||
145 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | 173 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
146 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | 174 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
@@ -164,11 +192,18 @@ class Git(FetchMethod): | |||
164 | ud.nocheckout = 1 | 192 | ud.nocheckout = 1 |
165 | 193 | ||
166 | ud.unresolvedrev = {} | 194 | ud.unresolvedrev = {} |
167 | branches = ud.parm.get("branch", "master").split(',') | 195 | branches = ud.parm.get("branch", "").split(',') |
196 | if branches == [""] and not ud.nobranch: | ||
197 | bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url) | ||
198 | branches = ["master"] | ||
168 | if len(branches) != len(ud.names): | 199 | if len(branches) != len(ud.names): |
169 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) | 200 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) |
170 | 201 | ||
171 | ud.cloneflags = "-s -n" | 202 | ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" |
203 | |||
204 | ud.cloneflags = "-n" | ||
205 | if not ud.noshared: | ||
206 | ud.cloneflags += " -s" | ||
172 | if ud.bareclone: | 207 | if ud.bareclone: |
173 | ud.cloneflags += " --mirror" | 208 | ud.cloneflags += " --mirror" |
174 | 209 | ||
@@ -227,7 +262,7 @@ class Git(FetchMethod): | |||
227 | for name in ud.names: | 262 | for name in ud.names: |
228 | ud.unresolvedrev[name] = 'HEAD' | 263 | ud.unresolvedrev[name] = 'HEAD' |
229 | 264 | ||
230 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" | 265 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all" |
231 | 266 | ||
232 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" | 267 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" |
233 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable | 268 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable |
@@ -236,20 +271,20 @@ class Git(FetchMethod): | |||
236 | ud.setup_revisions(d) | 271 | ud.setup_revisions(d) |
237 | 272 | ||
238 | for name in ud.names: | 273 | for name in ud.names: |
239 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one | 274 | # Ensure any revision that doesn't look like a SHA-1 is translated into one |
240 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): | 275 | if not sha1_re.match(ud.revisions[name] or ''): |
241 | if ud.revisions[name]: | 276 | if ud.revisions[name]: |
242 | ud.unresolvedrev[name] = ud.revisions[name] | 277 | ud.unresolvedrev[name] = ud.revisions[name] |
243 | ud.revisions[name] = self.latest_revision(ud, d, name) | 278 | ud.revisions[name] = self.latest_revision(ud, d, name) |
244 | 279 | ||
245 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) | 280 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) |
246 | if gitsrcname.startswith('.'): | 281 | if gitsrcname.startswith('.'): |
247 | gitsrcname = gitsrcname[1:] | 282 | gitsrcname = gitsrcname[1:] |
248 | 283 | ||
249 | # for rebaseable git repo, it is necessary to keep mirror tar ball | 284 | # For a rebaseable git repo, it is necessary to keep a mirror tar ball |
250 | # per revision, so that even the revision disappears from the | 285 | # per revision, so that even if the revision disappears from the |
251 | # upstream repo in the future, the mirror will remain intact and still | 286 | # upstream repo in the future, the mirror will remain intact and still |
252 | # contains the revision | 287 | # contain the revision |
253 | if ud.rebaseable: | 288 | if ud.rebaseable: |
254 | for name in ud.names: | 289 | for name in ud.names: |
255 | gitsrcname = gitsrcname + '_' + ud.revisions[name] | 290 | gitsrcname = gitsrcname + '_' + ud.revisions[name] |
@@ -293,7 +328,10 @@ class Git(FetchMethod): | |||
293 | return ud.clonedir | 328 | return ud.clonedir |
294 | 329 | ||
295 | def need_update(self, ud, d): | 330 | def need_update(self, ud, d): |
296 | return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) | 331 | return self.clonedir_need_update(ud, d) \ |
332 | or self.shallow_tarball_need_update(ud) \ | ||
333 | or self.tarball_need_update(ud) \ | ||
334 | or self.lfs_need_update(ud, d) | ||
297 | 335 | ||
298 | def clonedir_need_update(self, ud, d): | 336 | def clonedir_need_update(self, ud, d): |
299 | if not os.path.exists(ud.clonedir): | 337 | if not os.path.exists(ud.clonedir): |
@@ -305,6 +343,15 @@ class Git(FetchMethod): | |||
305 | return True | 343 | return True |
306 | return False | 344 | return False |
307 | 345 | ||
346 | def lfs_need_update(self, ud, d): | ||
347 | if self.clonedir_need_update(ud, d): | ||
348 | return True | ||
349 | |||
350 | for name in ud.names: | ||
351 | if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): | ||
352 | return True | ||
353 | return False | ||
354 | |||
308 | def clonedir_need_shallow_revs(self, ud, d): | 355 | def clonedir_need_shallow_revs(self, ud, d): |
309 | for rev in ud.shallow_revs: | 356 | for rev in ud.shallow_revs: |
310 | try: | 357 | try: |
@@ -324,6 +371,16 @@ class Git(FetchMethod): | |||
324 | # is not possible | 371 | # is not possible |
325 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): | 372 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): |
326 | return True | 373 | return True |
374 | # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0 | ||
375 | # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then | ||
376 | # we need to try premirrors first as using upstream is destined to fail. | ||
377 | if not trusted_network(d, ud.url): | ||
378 | return True | ||
379 | # the following check is to ensure incremental fetch in downloads, this is | ||
380 | # because the premirror might be old and does not contain the new rev required, | ||
381 | # and this will cause a total removal and new clone. So if we can reach to | ||
382 | # network, we prefer upstream over premirror, though the premirror might contain | ||
383 | # the new rev. | ||
327 | if os.path.exists(ud.clonedir): | 384 | if os.path.exists(ud.clonedir): |
328 | return False | 385 | return False |
329 | return True | 386 | return True |
@@ -337,17 +394,54 @@ class Git(FetchMethod): | |||
337 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | 394 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): |
338 | ud.localpath = ud.fullshallow | 395 | ud.localpath = ud.fullshallow |
339 | return | 396 | return |
340 | elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): | 397 | elif os.path.exists(ud.fullmirror) and self.need_update(ud, d): |
341 | bb.utils.mkdirhier(ud.clonedir) | 398 | if not os.path.exists(ud.clonedir): |
342 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | 399 | bb.utils.mkdirhier(ud.clonedir) |
343 | 400 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | |
401 | else: | ||
402 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
403 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) | ||
404 | output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) | ||
405 | if 'mirror' in output: | ||
406 | runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) | ||
407 | runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) | ||
408 | fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) | ||
409 | runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) | ||
344 | repourl = self._get_repo_url(ud) | 410 | repourl = self._get_repo_url(ud) |
345 | 411 | ||
412 | needs_clone = False | ||
413 | if os.path.exists(ud.clonedir): | ||
414 | # The directory may exist, but not be the top level of a bare git | ||
415 | # repository in which case it needs to be deleted and re-cloned. | ||
416 | try: | ||
417 | # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel | ||
418 | output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) | ||
419 | toplevel = output.rstrip() | ||
420 | |||
421 | if not bb.utils.path_is_descendant(toplevel, ud.clonedir): | ||
422 | logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir) | ||
423 | needs_clone = True | ||
424 | except bb.fetch2.FetchError as e: | ||
425 | logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) | ||
426 | needs_clone = True | ||
427 | except FileNotFoundError as e: | ||
428 | logger.warning("%s", e) | ||
429 | needs_clone = True | ||
430 | |||
431 | if needs_clone: | ||
432 | shutil.rmtree(ud.clonedir) | ||
433 | else: | ||
434 | needs_clone = True | ||
435 | |||
346 | # If the repo still doesn't exist, fallback to cloning it | 436 | # If the repo still doesn't exist, fallback to cloning it |
347 | if not os.path.exists(ud.clonedir): | 437 | if needs_clone: |
348 | # We do this since git will use a "-l" option automatically for local urls where possible | 438 | # We do this since git will use a "-l" option automatically for local urls where possible, |
439 | # but it doesn't work when git/objects is a symlink, only works when it is a directory. | ||
349 | if repourl.startswith("file://"): | 440 | if repourl.startswith("file://"): |
350 | repourl = repourl[7:] | 441 | repourl_path = repourl[7:] |
442 | objects = os.path.join(repourl_path, 'objects') | ||
443 | if os.path.isdir(objects) and not os.path.islink(objects): | ||
444 | repourl = repourl_path | ||
351 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) | 445 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) |
352 | if ud.proto.lower() != 'file': | 446 | if ud.proto.lower() != 'file': |
353 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) | 447 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) |
@@ -361,7 +455,11 @@ class Git(FetchMethod): | |||
361 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) | 455 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
362 | 456 | ||
363 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) | 457 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) |
364 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | 458 | |
459 | if ud.nobranch: | ||
460 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | ||
461 | else: | ||
462 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) | ||
365 | if ud.proto.lower() != 'file': | 463 | if ud.proto.lower() != 'file': |
366 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | 464 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
367 | progresshandler = GitProgressHandler(d) | 465 | progresshandler = GitProgressHandler(d) |
@@ -384,17 +482,16 @@ class Git(FetchMethod): | |||
384 | if missing_rev: | 482 | if missing_rev: |
385 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 483 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
386 | 484 | ||
387 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | 485 | if self.lfs_need_update(ud, d): |
388 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | 486 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching |
389 | # of all LFS blobs needed at the the srcrev. | 487 | # of all LFS blobs needed at the srcrev. |
390 | # | 488 | # |
391 | # It would be nice to just do this inline here by running 'git-lfs fetch' | 489 | # It would be nice to just do this inline here by running 'git-lfs fetch' |
392 | # on the bare clonedir, but that operation requires a working copy on some | 490 | # on the bare clonedir, but that operation requires a working copy on some |
393 | # releases of Git LFS. | 491 | # releases of Git LFS. |
394 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 492 | with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir: |
395 | try: | ||
396 | # Do the checkout. This implicitly involves a Git LFS fetch. | 493 | # Do the checkout. This implicitly involves a Git LFS fetch. |
397 | self.unpack(ud, tmpdir, d) | 494 | Git.unpack(self, ud, tmpdir, d) |
398 | 495 | ||
399 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | 496 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into |
400 | # the bare clonedir. | 497 | # the bare clonedir. |
@@ -408,12 +505,24 @@ class Git(FetchMethod): | |||
408 | # Only do this if the unpack resulted in a .git/lfs directory being | 505 | # Only do this if the unpack resulted in a .git/lfs directory being |
409 | # created; this only happens if at least one blob needed to be | 506 | # created; this only happens if at least one blob needed to be |
410 | # downloaded. | 507 | # downloaded. |
411 | if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): | 508 | if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")): |
412 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) | 509 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir) |
413 | finally: | ||
414 | bb.utils.remove(tmpdir, recurse=True) | ||
415 | 510 | ||
416 | def build_mirror_data(self, ud, d): | 511 | def build_mirror_data(self, ud, d): |
512 | |||
513 | # Create as a temp file and move atomically into position to avoid races | ||
514 | @contextmanager | ||
515 | def create_atomic(filename): | ||
516 | fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) | ||
517 | try: | ||
518 | yield tfile | ||
519 | umask = os.umask(0o666) | ||
520 | os.umask(umask) | ||
521 | os.chmod(tfile, (0o666 & ~umask)) | ||
522 | os.rename(tfile, filename) | ||
523 | finally: | ||
524 | os.close(fd) | ||
525 | |||
417 | if ud.shallow and ud.write_shallow_tarballs: | 526 | if ud.shallow and ud.write_shallow_tarballs: |
418 | if not os.path.exists(ud.fullshallow): | 527 | if not os.path.exists(ud.fullshallow): |
419 | if os.path.islink(ud.fullshallow): | 528 | if os.path.islink(ud.fullshallow): |
@@ -424,7 +533,8 @@ class Git(FetchMethod): | |||
424 | self.clone_shallow_local(ud, shallowclone, d) | 533 | self.clone_shallow_local(ud, shallowclone, d) |
425 | 534 | ||
426 | logger.info("Creating tarball of git repository") | 535 | logger.info("Creating tarball of git repository") |
427 | runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) | 536 | with create_atomic(ud.fullshallow) as tfile: |
537 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) | ||
428 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | 538 | runfetchcmd("touch %s.done" % ud.fullshallow, d) |
429 | finally: | 539 | finally: |
430 | bb.utils.remove(tempdir, recurse=True) | 540 | bb.utils.remove(tempdir, recurse=True) |
@@ -433,7 +543,11 @@ class Git(FetchMethod): | |||
433 | os.unlink(ud.fullmirror) | 543 | os.unlink(ud.fullmirror) |
434 | 544 | ||
435 | logger.info("Creating tarball of git repository") | 545 | logger.info("Creating tarball of git repository") |
436 | runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) | 546 | with create_atomic(ud.fullmirror) as tfile: |
547 | mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d, | ||
548 | quiet=True, workdir=ud.clonedir) | ||
549 | runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ." | ||
550 | % (tfile, mtime), d, workdir=ud.clonedir) | ||
437 | runfetchcmd("touch %s.done" % ud.fullmirror, d) | 551 | runfetchcmd("touch %s.done" % ud.fullmirror, d) |
438 | 552 | ||
439 | def clone_shallow_local(self, ud, dest, d): | 553 | def clone_shallow_local(self, ud, dest, d): |
@@ -495,18 +609,31 @@ class Git(FetchMethod): | |||
495 | def unpack(self, ud, destdir, d): | 609 | def unpack(self, ud, destdir, d): |
496 | """ unpack the downloaded src to destdir""" | 610 | """ unpack the downloaded src to destdir""" |
497 | 611 | ||
498 | subdir = ud.parm.get("subpath", "") | 612 | subdir = ud.parm.get("subdir") |
499 | if subdir != "": | 613 | subpath = ud.parm.get("subpath") |
500 | readpathspec = ":%s" % subdir | 614 | readpathspec = "" |
501 | def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) | 615 | def_destsuffix = "git/" |
502 | else: | 616 | |
503 | readpathspec = "" | 617 | if subpath: |
504 | def_destsuffix = "git/" | 618 | readpathspec = ":%s" % subpath |
619 | def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/')) | ||
620 | |||
621 | if subdir: | ||
622 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | ||
623 | if os.path.isabs(subdir): | ||
624 | if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)): | ||
625 | raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url) | ||
626 | destdir = subdir | ||
627 | else: | ||
628 | destdir = os.path.join(destdir, subdir) | ||
629 | def_destsuffix = "" | ||
505 | 630 | ||
506 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) | 631 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) |
507 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | 632 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
508 | if os.path.exists(destdir): | 633 | if os.path.exists(destdir): |
509 | bb.utils.prunedir(destdir) | 634 | bb.utils.prunedir(destdir) |
635 | if not ud.bareclone: | ||
636 | ud.unpack_tracer.unpack("git", destdir) | ||
510 | 637 | ||
511 | need_lfs = self._need_lfs(ud) | 638 | need_lfs = self._need_lfs(ud) |
512 | 639 | ||
@@ -516,13 +643,12 @@ class Git(FetchMethod): | |||
516 | source_found = False | 643 | source_found = False |
517 | source_error = [] | 644 | source_error = [] |
518 | 645 | ||
519 | if not source_found: | 646 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) |
520 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) | 647 | if clonedir_is_up_to_date: |
521 | if clonedir_is_up_to_date: | 648 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) |
522 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) | 649 | source_found = True |
523 | source_found = True | 650 | else: |
524 | else: | 651 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) |
525 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) | ||
526 | 652 | ||
527 | if not source_found: | 653 | if not source_found: |
528 | if ud.shallow: | 654 | if ud.shallow: |
@@ -546,9 +672,11 @@ class Git(FetchMethod): | |||
546 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) | 672 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) |
547 | elif not need_lfs: | 673 | elif not need_lfs: |
548 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) | 674 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) |
675 | else: | ||
676 | runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) | ||
549 | 677 | ||
550 | if not ud.nocheckout: | 678 | if not ud.nocheckout: |
551 | if subdir != "": | 679 | if subpath: |
552 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, | 680 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, |
553 | workdir=destdir) | 681 | workdir=destdir) |
554 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) | 682 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) |
@@ -597,6 +725,35 @@ class Git(FetchMethod): | |||
597 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 725 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
598 | return output.split()[0] != "0" | 726 | return output.split()[0] != "0" |
599 | 727 | ||
728 | def _lfs_objects_downloaded(self, ud, d, name, wd): | ||
729 | """ | ||
730 | Verifies whether the LFS objects for requested revisions have already been downloaded | ||
731 | """ | ||
732 | # Bail out early if this repository doesn't use LFS | ||
733 | if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): | ||
734 | return True | ||
735 | |||
736 | # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file | ||
737 | # existence. | ||
738 | # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git | ||
739 | cmd = "%s lfs ls-files -l %s" \ | ||
740 | % (ud.basecmd, ud.revisions[name]) | ||
741 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() | ||
742 | # Do not do any further matching if no objects are managed by LFS | ||
743 | if not output: | ||
744 | return True | ||
745 | |||
746 | # Match all lines beginning with the hexadecimal OID | ||
747 | oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") | ||
748 | for line in output.split("\n"): | ||
749 | oid = re.search(oid_regex, line) | ||
750 | if not oid: | ||
751 | bb.warn("git lfs ls-files output '%s' did not match expected format." % line) | ||
752 | if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): | ||
753 | return False | ||
754 | |||
755 | return True | ||
756 | |||
600 | def _need_lfs(self, ud): | 757 | def _need_lfs(self, ud): |
601 | return ud.parm.get("lfs", "1") == "1" | 758 | return ud.parm.get("lfs", "1") == "1" |
602 | 759 | ||
@@ -605,13 +762,11 @@ class Git(FetchMethod): | |||
605 | Check if the repository has 'lfs' (large file) content | 762 | Check if the repository has 'lfs' (large file) content |
606 | """ | 763 | """ |
607 | 764 | ||
608 | if not ud.nobranch: | 765 | if ud.nobranch: |
609 | branchname = ud.branches[ud.names[0]] | 766 | # If no branch is specified, use the current git commit |
610 | else: | 767 | refname = self._build_revision(ud, d, ud.names[0]) |
611 | branchname = "master" | 768 | elif wd == ud.clonedir: |
612 | 769 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | |
613 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | ||
614 | if wd == ud.clonedir: | ||
615 | refname = ud.branches[ud.names[0]] | 770 | refname = ud.branches[ud.names[0]] |
616 | else: | 771 | else: |
617 | refname = "origin/%s" % ud.branches[ud.names[0]] | 772 | refname = "origin/%s" % ud.branches[ud.names[0]] |
@@ -654,7 +809,6 @@ class Git(FetchMethod): | |||
654 | Return a unique key for the url | 809 | Return a unique key for the url |
655 | """ | 810 | """ |
656 | # Collapse adjacent slashes | 811 | # Collapse adjacent slashes |
657 | slash_re = re.compile(r"/+") | ||
658 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] | 812 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] |
659 | 813 | ||
660 | def _lsremote(self, ud, d, search): | 814 | def _lsremote(self, ud, d, search): |
@@ -687,6 +841,12 @@ class Git(FetchMethod): | |||
687 | """ | 841 | """ |
688 | Compute the HEAD revision for the url | 842 | Compute the HEAD revision for the url |
689 | """ | 843 | """ |
844 | if not d.getVar("__BBSRCREV_SEEN"): | ||
845 | raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path)) | ||
846 | |||
847 | # Ensure we mark as not cached | ||
848 | bb.fetch2.mark_recipe_nocache(d) | ||
849 | |||
690 | output = self._lsremote(ud, d, "") | 850 | output = self._lsremote(ud, d, "") |
691 | # Tags of the form ^{} may not work, need to fallback to other form | 851 | # Tags of the form ^{} may not work, need to fallback to other form |
692 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: | 852 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: |
@@ -711,38 +871,42 @@ class Git(FetchMethod): | |||
711 | """ | 871 | """ |
712 | pupver = ('', '') | 872 | pupver = ('', '') |
713 | 873 | ||
714 | tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
715 | try: | 874 | try: |
716 | output = self._lsremote(ud, d, "refs/tags/*") | 875 | output = self._lsremote(ud, d, "refs/tags/*") |
717 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: | 876 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: |
718 | bb.note("Could not list remote: %s" % str(e)) | 877 | bb.note("Could not list remote: %s" % str(e)) |
719 | return pupver | 878 | return pupver |
720 | 879 | ||
880 | rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)") | ||
881 | pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
882 | nonrel_re = re.compile(r"(alpha|beta|rc|final)+") | ||
883 | |||
721 | verstring = "" | 884 | verstring = "" |
722 | revision = "" | ||
723 | for line in output.split("\n"): | 885 | for line in output.split("\n"): |
724 | if not line: | 886 | if not line: |
725 | break | 887 | break |
726 | 888 | ||
727 | tag_head = line.split("/")[-1] | 889 | m = rev_tag_re.match(line) |
890 | if not m: | ||
891 | continue | ||
892 | |||
893 | (revision, tag) = m.groups() | ||
894 | |||
728 | # Ignore non-released branches | 895 | # Ignore non-released branches |
729 | m = re.search(r"(alpha|beta|rc|final)+", tag_head) | 896 | if nonrel_re.search(tag): |
730 | if m: | ||
731 | continue | 897 | continue |
732 | 898 | ||
733 | # search for version in the line | 899 | # search for version in the line |
734 | tag = tagregex.search(tag_head) | 900 | m = pver_re.search(tag) |
735 | if tag is None: | 901 | if not m: |
736 | continue | 902 | continue |
737 | 903 | ||
738 | tag = tag.group('pver') | 904 | pver = m.group('pver').replace("_", ".") |
739 | tag = tag.replace("_", ".") | ||
740 | 905 | ||
741 | if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: | 906 | if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0: |
742 | continue | 907 | continue |
743 | 908 | ||
744 | verstring = tag | 909 | verstring = pver |
745 | revision = line.split()[0] | ||
746 | pupver = (verstring, revision) | 910 | pupver = (verstring, revision) |
747 | 911 | ||
748 | return pupver | 912 | return pupver |