From 581233a798434f178898f12fa35bbcccb92e7768 Mon Sep 17 00:00:00 2001 From: Devendra Tewari Date: Mon, 19 Apr 2021 11:23:58 -0300 Subject: bitbake: lib/bb: Add bb.utils.rename() helper function and use for renaming os.rename can fail for example an incremental build in Docker fails with: OSError: [Errno 18] Invalid cross-device link when source and destination are on different overlay filesystems. Rather than trying to fix every call site, add a wrapper in bb.utils for renames. We can then handle cross device failures and fall back to shutil.move. The reason os.rename is still used is because shutil.move is too slow for speed sensitive sections of code. [YOCTO #14301] (Bitbake rev: c5c4e49574ab2a65e06298a0a77bb98b041cf56b) Signed-off-by: Devendra Tewari Signed-off-by: Richard Purdie --- bitbake/lib/bb/siggen.py | 2 +- bitbake/lib/bb/tests/fetch.py | 4 ++-- bitbake/lib/bb/utils.py | 18 +++++++++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index f3fa3000fa..07692e673c 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -402,7 +402,7 @@ class SignatureGeneratorBasic(SignatureGenerator): p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) - os.rename(tmpfile, sigfile) + bb.utils.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 0f7585e119..9291ce4a06 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -1798,7 +1798,7 @@ class GitShallowTest(FetcherTest): # Set up the mirror mirrordir = os.path.join(self.tempdir, 'mirror') - os.rename(self.dldir, mirrordir) + bb.utils.rename(self.dldir, mirrordir) self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) # Fetch from the mirror @@ -1916,7 +1916,7 @@ class GitShallowTest(FetcherTest): bb.utils.mkdirhier(mirrordir) self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir) - os.rename(os.path.join(self.dldir, mirrortarball), + bb.utils.rename(os.path.join(self.dldir, mirrortarball), os.path.join(mirrordir, mirrortarball)) # Fetch from the mirror diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index b282d09abf..6ba1d2a376 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -782,7 +782,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]: try: - os.rename(src, destpath) + bb.utils.rename(src, destpath) renamefailed = 0 except Exception as e: if e.errno != errno.EXDEV: @@ -796,7 +796,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if stat.S_ISREG(sstat[stat.ST_MODE]): try: # For safety copy then move it over. shutil.copyfile(src, destpath + "#new") - os.rename(destpath + "#new", destpath) + bb.utils.rename(destpath + "#new", destpath) didcopy = 1 except Exception as e: print('movefile: copy', src, '->', dest, 'failed.', e) @@ -874,7 +874,7 @@ def copyfile(src, dest, newmtime = None, sstat = None): # For safety copy then move it over. shutil.copyfile(src, dest + "#new") - os.rename(dest + "#new", dest) + bb.utils.rename(dest + "#new", dest) except Exception as e: logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e)) return False @@ -1669,3 +1669,15 @@ def is_semver(version): return False return True + +# Wrapper around os.rename which can handle cross device problems +# e.g. from container filesystems +def rename(src, dst): + try: + os.rename(src, dst) + except OSError as err: + if err.errno == 18: + # Invalid cross-device link error + shutil.move(src, dst) + else: + raise err -- cgit v1.2.3-54-g00ecf