From f5fd1c2fd3d50adcc0039b30694dd8e7e4070bd4 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 25 Oct 2021 22:15:48 +0800 Subject: bitbake: fetch2/git: Avoid races over mirror tarball creation There is a potential race over the mirror tarballs where a partial git repo could be extracted causing fetcher failures if the tarball is being rewritten whilst another build accesses it. Create the mirror tarball atomically to avoid this. [YOCTO #14441] (Bitbake rev: 7c8f344b81b8f8936214f87f695e24dc4e546659) Signed-off-by: Richard Purdie (cherry picked from commit 3250bc950c56bd7dd2114df26e5a8e13b04ceac8) Signed-off-by: Anuj Mittal Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/git.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index cf7424ebf4..95cd971d56 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -68,6 +68,7 @@ import subprocess import tempfile import bb import bb.progress +from contextlib import contextmanager from bb.fetch2 import FetchMethod from bb.fetch2 import runfetchcmd from bb.fetch2 import logger @@ -414,6 +415,20 @@ class Git(FetchMethod): bb.utils.remove(tmpdir, recurse=True) def build_mirror_data(self, ud, d): + + # Create as a temp file and move atomically into position to avoid races + @contextmanager + def create_atomic(filename, d): + fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) + try: + yield tfile + umask = os.umask(0o666) + os.umask(umask) + os.chmod(tfile, (0o666 & ~umask)) + runfetchcmd("mv %s %s" % (tfile, filename), d) + finally: + os.close(fd) + if ud.shallow and ud.write_shallow_tarballs: if not os.path.exists(ud.fullshallow): if os.path.islink(ud.fullshallow): @@ -424,7 +439,8 @@ class Git(FetchMethod): self.clone_shallow_local(ud, shallowclone, d) logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) + with create_atomic(ud.fullshallow, d) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) runfetchcmd("touch %s.done" % ud.fullshallow, d) finally: bb.utils.remove(tempdir, recurse=True) @@ -433,7 +449,8 @@ class Git(FetchMethod): os.unlink(ud.fullmirror) logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) + with create_atomic(ud.fullmirror, d) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir) runfetchcmd("touch %s.done" % ud.fullmirror, d) def clone_shallow_local(self, ud, dest, d): -- cgit v1.2.3-54-g00ecf