diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 54 |
1 files changed, 43 insertions, 11 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 8740e9c05f..cad1ae8207 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -44,7 +44,8 @@ Supported SRC_URI options are: | |||
44 | 44 | ||
45 | - nobranch | 45 | - nobranch |
46 | Don't check the SHA validation for branch. set this option for the recipe | 46 | Don't check the SHA validation for branch. set this option for the recipe |
47 | referring to commit which is valid in tag instead of branch. | 47 | referring to commit which is valid in any namespace (branch, tag, ...) |
48 | instead of branch. | ||
48 | The default is "0", set nobranch=1 if needed. | 49 | The default is "0", set nobranch=1 if needed. |
49 | 50 | ||
50 | - usehead | 51 | - usehead |
@@ -63,10 +64,12 @@ import errno | |||
63 | import fnmatch | 64 | import fnmatch |
64 | import os | 65 | import os |
65 | import re | 66 | import re |
67 | import shlex | ||
66 | import subprocess | 68 | import subprocess |
67 | import tempfile | 69 | import tempfile |
68 | import bb | 70 | import bb |
69 | import bb.progress | 71 | import bb.progress |
72 | from contextlib import contextmanager | ||
70 | from bb.fetch2 import FetchMethod | 73 | from bb.fetch2 import FetchMethod |
71 | from bb.fetch2 import runfetchcmd | 74 | from bb.fetch2 import runfetchcmd |
72 | from bb.fetch2 import logger | 75 | from bb.fetch2 import logger |
@@ -140,6 +143,10 @@ class Git(FetchMethod): | |||
140 | ud.proto = 'file' | 143 | ud.proto = 'file' |
141 | else: | 144 | else: |
142 | ud.proto = "git" | 145 | ud.proto = "git" |
146 | if ud.host == "github.com" and ud.proto == "git": | ||
147 | # github stopped supporting git protocol | ||
148 | # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git | ||
149 | ud.proto = "https" | ||
143 | 150 | ||
144 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | 151 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
145 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | 152 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
@@ -219,7 +226,12 @@ class Git(FetchMethod): | |||
219 | ud.shallow = False | 226 | ud.shallow = False |
220 | 227 | ||
221 | if ud.usehead: | 228 | if ud.usehead: |
222 | ud.unresolvedrev['default'] = 'HEAD' | 229 | # When usehead is set let's associate 'HEAD' with the unresolved |
230 | # rev of this repository. This will get resolved into a revision | ||
231 | # later. If an actual revision happens to have also been provided | ||
232 | # then this setting will be overridden. | ||
233 | for name in ud.names: | ||
234 | ud.unresolvedrev[name] = 'HEAD' | ||
223 | 235 | ||
224 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" | 236 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" |
225 | 237 | ||
@@ -342,7 +354,7 @@ class Git(FetchMethod): | |||
342 | # We do this since git will use a "-l" option automatically for local urls where possible | 354 | # We do this since git will use a "-l" option automatically for local urls where possible |
343 | if repourl.startswith("file://"): | 355 | if repourl.startswith("file://"): |
344 | repourl = repourl[7:] | 356 | repourl = repourl[7:] |
345 | clone_cmd = "LANG=C %s clone --bare --mirror \"%s\" %s --progress" % (ud.basecmd, repourl, ud.clonedir) | 357 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) |
346 | if ud.proto.lower() != 'file': | 358 | if ud.proto.lower() != 'file': |
347 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) | 359 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) |
348 | progresshandler = GitProgressHandler(d) | 360 | progresshandler = GitProgressHandler(d) |
@@ -354,8 +366,12 @@ class Git(FetchMethod): | |||
354 | if "origin" in output: | 366 | if "origin" in output: |
355 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) | 367 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
356 | 368 | ||
357 | runfetchcmd("%s remote add --mirror=fetch origin \"%s\"" % (ud.basecmd, repourl), d, workdir=ud.clonedir) | 369 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) |
358 | fetch_cmd = "LANG=C %s fetch -f --progress \"%s\" refs/*:refs/*" % (ud.basecmd, repourl) | 370 | |
371 | if ud.nobranch: | ||
372 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | ||
373 | else: | ||
374 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) | ||
359 | if ud.proto.lower() != 'file': | 375 | if ud.proto.lower() != 'file': |
360 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | 376 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
361 | progresshandler = GitProgressHandler(d) | 377 | progresshandler = GitProgressHandler(d) |
@@ -388,7 +404,7 @@ class Git(FetchMethod): | |||
388 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 404 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) |
389 | try: | 405 | try: |
390 | # Do the checkout. This implicitly involves a Git LFS fetch. | 406 | # Do the checkout. This implicitly involves a Git LFS fetch. |
391 | self.unpack(ud, tmpdir, d) | 407 | Git.unpack(self, ud, tmpdir, d) |
392 | 408 | ||
393 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | 409 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into |
394 | # the bare clonedir. | 410 | # the bare clonedir. |
@@ -408,6 +424,20 @@ class Git(FetchMethod): | |||
408 | bb.utils.remove(tmpdir, recurse=True) | 424 | bb.utils.remove(tmpdir, recurse=True) |
409 | 425 | ||
410 | def build_mirror_data(self, ud, d): | 426 | def build_mirror_data(self, ud, d): |
427 | |||
428 | # Create as a temp file and move atomically into position to avoid races | ||
429 | @contextmanager | ||
430 | def create_atomic(filename): | ||
431 | fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) | ||
432 | try: | ||
433 | yield tfile | ||
434 | umask = os.umask(0o666) | ||
435 | os.umask(umask) | ||
436 | os.chmod(tfile, (0o666 & ~umask)) | ||
437 | os.rename(tfile, filename) | ||
438 | finally: | ||
439 | os.close(fd) | ||
440 | |||
411 | if ud.shallow and ud.write_shallow_tarballs: | 441 | if ud.shallow and ud.write_shallow_tarballs: |
412 | if not os.path.exists(ud.fullshallow): | 442 | if not os.path.exists(ud.fullshallow): |
413 | if os.path.islink(ud.fullshallow): | 443 | if os.path.islink(ud.fullshallow): |
@@ -418,7 +448,8 @@ class Git(FetchMethod): | |||
418 | self.clone_shallow_local(ud, shallowclone, d) | 448 | self.clone_shallow_local(ud, shallowclone, d) |
419 | 449 | ||
420 | logger.info("Creating tarball of git repository") | 450 | logger.info("Creating tarball of git repository") |
421 | runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) | 451 | with create_atomic(ud.fullshallow) as tfile: |
452 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) | ||
422 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | 453 | runfetchcmd("touch %s.done" % ud.fullshallow, d) |
423 | finally: | 454 | finally: |
424 | bb.utils.remove(tempdir, recurse=True) | 455 | bb.utils.remove(tempdir, recurse=True) |
@@ -427,7 +458,8 @@ class Git(FetchMethod): | |||
427 | os.unlink(ud.fullmirror) | 458 | os.unlink(ud.fullmirror) |
428 | 459 | ||
429 | logger.info("Creating tarball of git repository") | 460 | logger.info("Creating tarball of git repository") |
430 | runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) | 461 | with create_atomic(ud.fullmirror) as tfile: |
462 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir) | ||
431 | runfetchcmd("touch %s.done" % ud.fullmirror, d) | 463 | runfetchcmd("touch %s.done" % ud.fullmirror, d) |
432 | 464 | ||
433 | def clone_shallow_local(self, ud, dest, d): | 465 | def clone_shallow_local(self, ud, dest, d): |
@@ -533,7 +565,7 @@ class Git(FetchMethod): | |||
533 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) | 565 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) |
534 | 566 | ||
535 | repourl = self._get_repo_url(ud) | 567 | repourl = self._get_repo_url(ud) |
536 | runfetchcmd("%s remote set-url origin \"%s\"" % (ud.basecmd, repourl), d, workdir=destdir) | 568 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) |
537 | 569 | ||
538 | if self._contains_lfs(ud, d, destdir): | 570 | if self._contains_lfs(ud, d, destdir): |
539 | if need_lfs and not self._find_git_lfs(d): | 571 | if need_lfs and not self._find_git_lfs(d): |
@@ -661,8 +693,8 @@ class Git(FetchMethod): | |||
661 | d.setVar('_BB_GIT_IN_LSREMOTE', '1') | 693 | d.setVar('_BB_GIT_IN_LSREMOTE', '1') |
662 | try: | 694 | try: |
663 | repourl = self._get_repo_url(ud) | 695 | repourl = self._get_repo_url(ud) |
664 | cmd = "%s ls-remote \"%s\" %s" % \ | 696 | cmd = "%s ls-remote %s %s" % \ |
665 | (ud.basecmd, repourl, search) | 697 | (ud.basecmd, shlex.quote(repourl), search) |
666 | if ud.proto.lower() != 'file': | 698 | if ud.proto.lower() != 'file': |
667 | bb.fetch2.check_network_access(d, cmd, repourl) | 699 | bb.fetch2.check_network_access(d, cmd, repourl) |
668 | output = runfetchcmd(cmd, d, True) | 700 | output = runfetchcmd(cmd, d, True) |