summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r--bitbake/lib/bb/fetch2/git.py294
1 files changed, 229 insertions, 65 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e3ba80a3f5..c7ff769fdf 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -44,13 +44,27 @@ Supported SRC_URI options are:
44 44
45- nobranch 45- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe 46 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch. 47 referring to commit which is valid in any namespace (branch, tag, ...)
48 instead of branch.
48 The default is "0", set nobranch=1 if needed. 49 The default is "0", set nobranch=1 if needed.
49 50
51- subpath
52 Limit the checkout to a specific subpath of the tree.
53 By default, checkout the whole tree, set subpath=<path> if needed
54
55- destsuffix
56 The name of the path in which to place the checkout.
57 By default, the path is git/, set destsuffix=<suffix> if needed
58
50- usehead 59- usehead
51 For local git:// urls to use the current branch HEAD as the revision for use with 60 For local git:// urls to use the current branch HEAD as the revision for use with
52 AUTOREV. Implies nobranch. 61 AUTOREV. Implies nobranch.
53 62
63- lfs
64 Enable the checkout to use LFS for large files. This will download all LFS files
65 in the download step, as the unpack step does not have network access.
66 The default is "1", set lfs=0 to skip.
67
54""" 68"""
55 69
56# Copyright (C) 2005 Richard Purdie 70# Copyright (C) 2005 Richard Purdie
@@ -64,14 +78,20 @@ import fnmatch
64import os 78import os
65import re 79import re
66import shlex 80import shlex
81import shutil
67import subprocess 82import subprocess
68import tempfile 83import tempfile
69import bb 84import bb
70import bb.progress 85import bb.progress
86from contextlib import contextmanager
71from bb.fetch2 import FetchMethod 87from bb.fetch2 import FetchMethod
72from bb.fetch2 import runfetchcmd 88from bb.fetch2 import runfetchcmd
73from bb.fetch2 import logger 89from bb.fetch2 import logger
90from bb.fetch2 import trusted_network
91
74 92
93sha1_re = re.compile(r'^[0-9a-f]{40}$')
94slash_re = re.compile(r"/+")
75 95
76class GitProgressHandler(bb.progress.LineFilterProgressHandler): 96class GitProgressHandler(bb.progress.LineFilterProgressHandler):
77 """Extract progress information from git output""" 97 """Extract progress information from git output"""
@@ -130,6 +150,9 @@ class Git(FetchMethod):
130 def supports_checksum(self, urldata): 150 def supports_checksum(self, urldata):
131 return False 151 return False
132 152
153 def cleanup_upon_failure(self):
154 return False
155
133 def urldata_init(self, ud, d): 156 def urldata_init(self, ud, d):
134 """ 157 """
135 init git specific variable within url data 158 init git specific variable within url data
@@ -141,6 +164,11 @@ class Git(FetchMethod):
141 ud.proto = 'file' 164 ud.proto = 'file'
142 else: 165 else:
143 ud.proto = "git" 166 ud.proto = "git"
167 if ud.host == "github.com" and ud.proto == "git":
168 # github stopped supporting git protocol
169 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
170 ud.proto = "https"
171 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
144 172
145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): 173 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) 174 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -164,11 +192,18 @@ class Git(FetchMethod):
164 ud.nocheckout = 1 192 ud.nocheckout = 1
165 193
166 ud.unresolvedrev = {} 194 ud.unresolvedrev = {}
167 branches = ud.parm.get("branch", "master").split(',') 195 branches = ud.parm.get("branch", "").split(',')
196 if branches == [""] and not ud.nobranch:
197 bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
198 branches = ["master"]
168 if len(branches) != len(ud.names): 199 if len(branches) != len(ud.names):
169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) 200 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
170 201
171 ud.cloneflags = "-s -n" 202 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
203
204 ud.cloneflags = "-n"
205 if not ud.noshared:
206 ud.cloneflags += " -s"
172 if ud.bareclone: 207 if ud.bareclone:
173 ud.cloneflags += " --mirror" 208 ud.cloneflags += " --mirror"
174 209
@@ -227,7 +262,7 @@ class Git(FetchMethod):
227 for name in ud.names: 262 for name in ud.names:
228 ud.unresolvedrev[name] = 'HEAD' 263 ud.unresolvedrev[name] = 'HEAD'
229 264
230 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" 265 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all"
231 266
232 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" 267 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
233 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable 268 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -236,20 +271,20 @@ class Git(FetchMethod):
236 ud.setup_revisions(d) 271 ud.setup_revisions(d)
237 272
238 for name in ud.names: 273 for name in ud.names:
239 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one 274 # Ensure any revision that doesn't look like a SHA-1 is translated into one
240 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): 275 if not sha1_re.match(ud.revisions[name] or ''):
241 if ud.revisions[name]: 276 if ud.revisions[name]:
242 ud.unresolvedrev[name] = ud.revisions[name] 277 ud.unresolvedrev[name] = ud.revisions[name]
243 ud.revisions[name] = self.latest_revision(ud, d, name) 278 ud.revisions[name] = self.latest_revision(ud, d, name)
244 279
245 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) 280 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
246 if gitsrcname.startswith('.'): 281 if gitsrcname.startswith('.'):
247 gitsrcname = gitsrcname[1:] 282 gitsrcname = gitsrcname[1:]
248 283
249 # for rebaseable git repo, it is necessary to keep mirror tar ball 284 # For a rebaseable git repo, it is necessary to keep a mirror tar ball
250 # per revision, so that even the revision disappears from the 285 # per revision, so that even if the revision disappears from the
251 # upstream repo in the future, the mirror will remain intact and still 286 # upstream repo in the future, the mirror will remain intact and still
252 # contains the revision 287 # contain the revision
253 if ud.rebaseable: 288 if ud.rebaseable:
254 for name in ud.names: 289 for name in ud.names:
255 gitsrcname = gitsrcname + '_' + ud.revisions[name] 290 gitsrcname = gitsrcname + '_' + ud.revisions[name]
@@ -293,7 +328,10 @@ class Git(FetchMethod):
293 return ud.clonedir 328 return ud.clonedir
294 329
295 def need_update(self, ud, d): 330 def need_update(self, ud, d):
296 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) 331 return self.clonedir_need_update(ud, d) \
332 or self.shallow_tarball_need_update(ud) \
333 or self.tarball_need_update(ud) \
334 or self.lfs_need_update(ud, d)
297 335
298 def clonedir_need_update(self, ud, d): 336 def clonedir_need_update(self, ud, d):
299 if not os.path.exists(ud.clonedir): 337 if not os.path.exists(ud.clonedir):
@@ -305,6 +343,15 @@ class Git(FetchMethod):
305 return True 343 return True
306 return False 344 return False
307 345
346 def lfs_need_update(self, ud, d):
347 if self.clonedir_need_update(ud, d):
348 return True
349
350 for name in ud.names:
351 if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
352 return True
353 return False
354
308 def clonedir_need_shallow_revs(self, ud, d): 355 def clonedir_need_shallow_revs(self, ud, d):
309 for rev in ud.shallow_revs: 356 for rev in ud.shallow_revs:
310 try: 357 try:
@@ -324,6 +371,16 @@ class Git(FetchMethod):
324 # is not possible 371 # is not possible
325 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): 372 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
326 return True 373 return True
374 # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
375 # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
376 # we need to try premirrors first as using upstream is destined to fail.
377 if not trusted_network(d, ud.url):
378 return True
379 # the following check is to ensure incremental fetch in downloads, this is
380 # because the premirror might be old and does not contain the new rev required,
381 # and this will cause a total removal and new clone. So if we can reach to
382 # network, we prefer upstream over premirror, though the premirror might contain
383 # the new rev.
327 if os.path.exists(ud.clonedir): 384 if os.path.exists(ud.clonedir):
328 return False 385 return False
329 return True 386 return True
@@ -337,17 +394,54 @@ class Git(FetchMethod):
337 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 394 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
338 ud.localpath = ud.fullshallow 395 ud.localpath = ud.fullshallow
339 return 396 return
340 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): 397 elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
341 bb.utils.mkdirhier(ud.clonedir) 398 if not os.path.exists(ud.clonedir):
342 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 399 bb.utils.mkdirhier(ud.clonedir)
343 400 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
401 else:
402 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
403 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
404 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
405 if 'mirror' in output:
406 runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
407 runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
408 fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd)
409 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
344 repourl = self._get_repo_url(ud) 410 repourl = self._get_repo_url(ud)
345 411
412 needs_clone = False
413 if os.path.exists(ud.clonedir):
414 # The directory may exist, but not be the top level of a bare git
415 # repository in which case it needs to be deleted and re-cloned.
416 try:
417 # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
418 output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
419 toplevel = output.rstrip()
420
421 if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
422 logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
423 needs_clone = True
424 except bb.fetch2.FetchError as e:
425 logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
426 needs_clone = True
427 except FileNotFoundError as e:
428 logger.warning("%s", e)
429 needs_clone = True
430
431 if needs_clone:
432 shutil.rmtree(ud.clonedir)
433 else:
434 needs_clone = True
435
346 # If the repo still doesn't exist, fallback to cloning it 436 # If the repo still doesn't exist, fallback to cloning it
347 if not os.path.exists(ud.clonedir): 437 if needs_clone:
348 # We do this since git will use a "-l" option automatically for local urls where possible 438 # We do this since git will use a "-l" option automatically for local urls where possible,
439 # but it doesn't work when git/objects is a symlink, only works when it is a directory.
349 if repourl.startswith("file://"): 440 if repourl.startswith("file://"):
350 repourl = repourl[7:] 441 repourl_path = repourl[7:]
442 objects = os.path.join(repourl_path, 'objects')
443 if os.path.isdir(objects) and not os.path.islink(objects):
444 repourl = repourl_path
351 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) 445 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
352 if ud.proto.lower() != 'file': 446 if ud.proto.lower() != 'file':
353 bb.fetch2.check_network_access(d, clone_cmd, ud.url) 447 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
@@ -361,7 +455,11 @@ class Git(FetchMethod):
361 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) 455 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
362 456
363 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) 457 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
364 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) 458
459 if ud.nobranch:
460 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
461 else:
462 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
365 if ud.proto.lower() != 'file': 463 if ud.proto.lower() != 'file':
366 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 464 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
367 progresshandler = GitProgressHandler(d) 465 progresshandler = GitProgressHandler(d)
@@ -384,17 +482,16 @@ class Git(FetchMethod):
384 if missing_rev: 482 if missing_rev:
385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 483 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
386 484
387 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): 485 if self.lfs_need_update(ud, d):
388 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching 486 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
389 # of all LFS blobs needed at the the srcrev. 487 # of all LFS blobs needed at the srcrev.
390 # 488 #
391 # It would be nice to just do this inline here by running 'git-lfs fetch' 489 # It would be nice to just do this inline here by running 'git-lfs fetch'
392 # on the bare clonedir, but that operation requires a working copy on some 490 # on the bare clonedir, but that operation requires a working copy on some
393 # releases of Git LFS. 491 # releases of Git LFS.
394 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 492 with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
395 try:
396 # Do the checkout. This implicitly involves a Git LFS fetch. 493 # Do the checkout. This implicitly involves a Git LFS fetch.
397 self.unpack(ud, tmpdir, d) 494 Git.unpack(self, ud, tmpdir, d)
398 495
399 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into 496 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
400 # the bare clonedir. 497 # the bare clonedir.
@@ -408,12 +505,24 @@ class Git(FetchMethod):
408 # Only do this if the unpack resulted in a .git/lfs directory being 505 # Only do this if the unpack resulted in a .git/lfs directory being
409 # created; this only happens if at least one blob needed to be 506 # created; this only happens if at least one blob needed to be
410 # downloaded. 507 # downloaded.
411 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): 508 if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
412 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) 509 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
413 finally:
414 bb.utils.remove(tmpdir, recurse=True)
415 510
416 def build_mirror_data(self, ud, d): 511 def build_mirror_data(self, ud, d):
512
513 # Create as a temp file and move atomically into position to avoid races
514 @contextmanager
515 def create_atomic(filename):
516 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
517 try:
518 yield tfile
519 umask = os.umask(0o666)
520 os.umask(umask)
521 os.chmod(tfile, (0o666 & ~umask))
522 os.rename(tfile, filename)
523 finally:
524 os.close(fd)
525
417 if ud.shallow and ud.write_shallow_tarballs: 526 if ud.shallow and ud.write_shallow_tarballs:
418 if not os.path.exists(ud.fullshallow): 527 if not os.path.exists(ud.fullshallow):
419 if os.path.islink(ud.fullshallow): 528 if os.path.islink(ud.fullshallow):
@@ -424,7 +533,8 @@ class Git(FetchMethod):
424 self.clone_shallow_local(ud, shallowclone, d) 533 self.clone_shallow_local(ud, shallowclone, d)
425 534
426 logger.info("Creating tarball of git repository") 535 logger.info("Creating tarball of git repository")
427 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) 536 with create_atomic(ud.fullshallow) as tfile:
537 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
428 runfetchcmd("touch %s.done" % ud.fullshallow, d) 538 runfetchcmd("touch %s.done" % ud.fullshallow, d)
429 finally: 539 finally:
430 bb.utils.remove(tempdir, recurse=True) 540 bb.utils.remove(tempdir, recurse=True)
@@ -433,7 +543,11 @@ class Git(FetchMethod):
433 os.unlink(ud.fullmirror) 543 os.unlink(ud.fullmirror)
434 544
435 logger.info("Creating tarball of git repository") 545 logger.info("Creating tarball of git repository")
436 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 546 with create_atomic(ud.fullmirror) as tfile:
547 mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
548 quiet=True, workdir=ud.clonedir)
549 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
550 % (tfile, mtime), d, workdir=ud.clonedir)
437 runfetchcmd("touch %s.done" % ud.fullmirror, d) 551 runfetchcmd("touch %s.done" % ud.fullmirror, d)
438 552
439 def clone_shallow_local(self, ud, dest, d): 553 def clone_shallow_local(self, ud, dest, d):
@@ -495,18 +609,31 @@ class Git(FetchMethod):
495 def unpack(self, ud, destdir, d): 609 def unpack(self, ud, destdir, d):
496 """ unpack the downloaded src to destdir""" 610 """ unpack the downloaded src to destdir"""
497 611
498 subdir = ud.parm.get("subpath", "") 612 subdir = ud.parm.get("subdir")
499 if subdir != "": 613 subpath = ud.parm.get("subpath")
500 readpathspec = ":%s" % subdir 614 readpathspec = ""
501 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) 615 def_destsuffix = "git/"
502 else: 616
503 readpathspec = "" 617 if subpath:
504 def_destsuffix = "git/" 618 readpathspec = ":%s" % subpath
619 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
620
621 if subdir:
622 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
623 if os.path.isabs(subdir):
624 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
625 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
626 destdir = subdir
627 else:
628 destdir = os.path.join(destdir, subdir)
629 def_destsuffix = ""
505 630
506 destsuffix = ud.parm.get("destsuffix", def_destsuffix) 631 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
507 destdir = ud.destdir = os.path.join(destdir, destsuffix) 632 destdir = ud.destdir = os.path.join(destdir, destsuffix)
508 if os.path.exists(destdir): 633 if os.path.exists(destdir):
509 bb.utils.prunedir(destdir) 634 bb.utils.prunedir(destdir)
635 if not ud.bareclone:
636 ud.unpack_tracer.unpack("git", destdir)
510 637
511 need_lfs = self._need_lfs(ud) 638 need_lfs = self._need_lfs(ud)
512 639
@@ -516,13 +643,12 @@ class Git(FetchMethod):
516 source_found = False 643 source_found = False
517 source_error = [] 644 source_error = []
518 645
519 if not source_found: 646 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
520 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) 647 if clonedir_is_up_to_date:
521 if clonedir_is_up_to_date: 648 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
522 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) 649 source_found = True
523 source_found = True 650 else:
524 else: 651 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
525 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
526 652
527 if not source_found: 653 if not source_found:
528 if ud.shallow: 654 if ud.shallow:
@@ -546,9 +672,11 @@ class Git(FetchMethod):
546 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) 672 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
547 elif not need_lfs: 673 elif not need_lfs:
548 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) 674 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
675 else:
676 runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
549 677
550 if not ud.nocheckout: 678 if not ud.nocheckout:
551 if subdir != "": 679 if subpath:
552 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 680 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
553 workdir=destdir) 681 workdir=destdir)
554 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 682 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
@@ -597,6 +725,35 @@ class Git(FetchMethod):
597 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 725 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
598 return output.split()[0] != "0" 726 return output.split()[0] != "0"
599 727
728 def _lfs_objects_downloaded(self, ud, d, name, wd):
729 """
730 Verifies whether the LFS objects for requested revisions have already been downloaded
731 """
732 # Bail out early if this repository doesn't use LFS
733 if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd):
734 return True
735
736 # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
737 # existence.
738 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
739 cmd = "%s lfs ls-files -l %s" \
740 % (ud.basecmd, ud.revisions[name])
741 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
742 # Do not do any further matching if no objects are managed by LFS
743 if not output:
744 return True
745
746 # Match all lines beginning with the hexadecimal OID
747 oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
748 for line in output.split("\n"):
749 oid = re.search(oid_regex, line)
750 if not oid:
751 bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
752 if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
753 return False
754
755 return True
756
600 def _need_lfs(self, ud): 757 def _need_lfs(self, ud):
601 return ud.parm.get("lfs", "1") == "1" 758 return ud.parm.get("lfs", "1") == "1"
602 759
@@ -605,13 +762,11 @@ class Git(FetchMethod):
605 Check if the repository has 'lfs' (large file) content 762 Check if the repository has 'lfs' (large file) content
606 """ 763 """
607 764
608 if not ud.nobranch: 765 if ud.nobranch:
609 branchname = ud.branches[ud.names[0]] 766 # If no branch is specified, use the current git commit
610 else: 767 refname = self._build_revision(ud, d, ud.names[0])
611 branchname = "master" 768 elif wd == ud.clonedir:
612 769 # The bare clonedir doesn't use the remote names; it has the branch immediately.
613 # The bare clonedir doesn't use the remote names; it has the branch immediately.
614 if wd == ud.clonedir:
615 refname = ud.branches[ud.names[0]] 770 refname = ud.branches[ud.names[0]]
616 else: 771 else:
617 refname = "origin/%s" % ud.branches[ud.names[0]] 772 refname = "origin/%s" % ud.branches[ud.names[0]]
@@ -654,7 +809,6 @@ class Git(FetchMethod):
654 Return a unique key for the url 809 Return a unique key for the url
655 """ 810 """
656 # Collapse adjacent slashes 811 # Collapse adjacent slashes
657 slash_re = re.compile(r"/+")
658 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] 812 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
659 813
660 def _lsremote(self, ud, d, search): 814 def _lsremote(self, ud, d, search):
@@ -687,6 +841,12 @@ class Git(FetchMethod):
687 """ 841 """
688 Compute the HEAD revision for the url 842 Compute the HEAD revision for the url
689 """ 843 """
844 if not d.getVar("__BBSRCREV_SEEN"):
845 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
846
847 # Ensure we mark as not cached
848 bb.fetch2.mark_recipe_nocache(d)
849
690 output = self._lsremote(ud, d, "") 850 output = self._lsremote(ud, d, "")
691 # Tags of the form ^{} may not work, need to fallback to other form 851 # Tags of the form ^{} may not work, need to fallback to other form
692 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 852 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
@@ -711,38 +871,42 @@ class Git(FetchMethod):
711 """ 871 """
712 pupver = ('', '') 872 pupver = ('', '')
713 873
714 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
715 try: 874 try:
716 output = self._lsremote(ud, d, "refs/tags/*") 875 output = self._lsremote(ud, d, "refs/tags/*")
717 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: 876 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
718 bb.note("Could not list remote: %s" % str(e)) 877 bb.note("Could not list remote: %s" % str(e))
719 return pupver 878 return pupver
720 879
880 rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
881 pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
882 nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
883
721 verstring = "" 884 verstring = ""
722 revision = ""
723 for line in output.split("\n"): 885 for line in output.split("\n"):
724 if not line: 886 if not line:
725 break 887 break
726 888
727 tag_head = line.split("/")[-1] 889 m = rev_tag_re.match(line)
890 if not m:
891 continue
892
893 (revision, tag) = m.groups()
894
728 # Ignore non-released branches 895 # Ignore non-released branches
729 m = re.search(r"(alpha|beta|rc|final)+", tag_head) 896 if nonrel_re.search(tag):
730 if m:
731 continue 897 continue
732 898
733 # search for version in the line 899 # search for version in the line
734 tag = tagregex.search(tag_head) 900 m = pver_re.search(tag)
735 if tag is None: 901 if not m:
736 continue 902 continue
737 903
738 tag = tag.group('pver') 904 pver = m.group('pver').replace("_", ".")
739 tag = tag.replace("_", ".")
740 905
741 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: 906 if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
742 continue 907 continue
743 908
744 verstring = tag 909 verstring = pver
745 revision = line.split()[0]
746 pupver = (verstring, revision) 910 pupver = (verstring, revision)
747 911
748 return pupver 912 return pupver