summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorJoshua Watt <JPEWhacker@gmail.com>2022-08-03 09:04:41 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-08-09 16:28:03 +0100
commitfdc9d9e4fc386e178800b15ea22a368749e90965 (patch)
tree134ba923e8aa1935c6a2056f031e18d0169ce826 /bitbake
parentc58fca98e4959684c5d3f3ac93570ff06898df58 (diff)
downloadpoky-fdc9d9e4fc386e178800b15ea22a368749e90965.tar.gz
bitbake: siggen: Fix insufficent entropy in sigtask file names
Signature generation uses mkstemp() to get a file descriptor to a unique file and then write the signature into it. However, the unique file name generation in glibc is based on the system timestamp, which means that with highly parallel builds it is more likely than one might expect expected that a conflict will occur between two different builder nodes. When operating over NFS (such as a shared sstate cache), this can cause race conditions and rare failures (particularly with NFS servers that may not correctly implement O_EXCL). The signature generation code is particularly susceptible to races since a single "sigtask." prefix used for all signatures from all tasks, which makes collision even more likely. To work around this, add an internal implementation of mkstemp() that adds additional truly random entropy to the file name to eliminate conflicts. (Bitbake rev: 97955f3c1c738aa4b4478a6ec10a08094ffc689d) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/siggen.py2
-rw-r--r--bitbake/lib/bb/utils.py21
2 files changed, 22 insertions, 1 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 3f3d6df54d..bb80343a97 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -425,7 +425,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
425 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid)) 425 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
426 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash) 426 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
427 427
428 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") 428 fd, tmpfile = bb.utils.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
429 try: 429 try:
430 with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f: 430 with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f:
431 json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder) 431 json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder)
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 19ed68ea62..b8b90df8d3 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -28,6 +28,8 @@ import signal
28import collections 28import collections
29import copy 29import copy
30import ctypes 30import ctypes
31import random
32import tempfile
31from subprocess import getstatusoutput 33from subprocess import getstatusoutput
32from contextlib import contextmanager 34from contextlib import contextmanager
33from ctypes import cdll 35from ctypes import cdll
@@ -1754,3 +1756,22 @@ def is_local_uid(uid=''):
1754 if str(uid) == line_split[2]: 1756 if str(uid) == line_split[2]:
1755 return True 1757 return True
1756 return False 1758 return False
1759
1760def mkstemp(suffix=None, prefix=None, dir=None, text=False):
1761 """
1762 Generates a unique filename, independent of time.
1763
1764 mkstemp() in glibc (at least) generates unique file names based on the
1765 current system time. When combined with highly parallel builds, and
1766 operating over NFS (e.g. shared sstate/downloads) this can result in
1767 conflicts and race conditions.
1768
1769 This function adds additional entropy to the file name so that a collision
1770 is independent of time and thus extremely unlikely.
1771 """
1772 entropy = "".join(random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890", k=20))
1773 if prefix:
1774 prefix = prefix + entropy
1775 else:
1776 prefix = tempfile.gettempprefix() + entropy
1777 return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)