From f619b134039765e86bc0dec86485953b623ee89f Mon Sep 17 00:00:00 2001 From: Aníbal Limón Date: Fri, 15 Nov 2019 08:47:23 -0600 Subject: bitbake: lib/bb: Add BB_SIGNATURE_LOCAL_DIRS_EXCLUDE to speed-up taskhash on directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new BB_SIGNATURE_LOCAL_DIRS_EXCLUDE allows you to specify a list of directories to exclude when making taskhash, our specific case is using SRC_URI that points local VCS directory. Use bb.fetch.module to set default to: "CVS .bzr .git .hg .osc .p4 .repo .svn" (Bitbake rev: 4c7d689341f471efdf8ab7c2a7c6a19d2d370f5c) Signed-off-by: Aníbal Limón Signed-off-by: Richard Purdie (cherry picked from commit 923aff060d8aba8456979c35b16d300ba7c13ff9) Signed-off-by: Armin Kuster Signed-off-by: Richard Purdie --- bitbake/lib/bb/checksum.py | 5 +++-- bitbake/lib/bb/fetch2/__init__.py | 4 ++-- bitbake/lib/bb/siggen.py | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/checksum.py b/bitbake/lib/bb/checksum.py index 5bc8a8fcb6..677020f497 100644 --- a/bitbake/lib/bb/checksum.py +++ b/bitbake/lib/bb/checksum.py @@ -74,7 +74,7 @@ class FileChecksumCache(MultiProcessCache): else: dest[0][h] = source[0][h] - def get_checksums(self, filelist, pn): + def get_checksums(self, filelist, pn, localdirsexclude): """Get checksums for a list of files""" def checksum_file(f): @@ -90,7 +90,8 @@ class FileChecksumCache(MultiProcessCache): if pth == "/": bb.fatal("Refusing to checksum /") dirchecksums = [] - for root, dirs, files in os.walk(pth): + for root, dirs, files in os.walk(pth, topdown=True): + [dirs.remove(d) for d in list(dirs) if d in localdirsexclude] for name in files: fullpth = os.path.join(root, name) checksum = checksum_file(fullpth) diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 07de6c2693..731c160892 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -1197,14 +1197,14 @@ def get_checksum_file_list(d): return " ".join(filelist) -def get_file_checksums(filelist, pn): +def get_file_checksums(filelist, pn, localdirsexclude): """Get a list of the checksums for a list of local files Returns the checksums for a list of local files, caching the results as it proceeds """ - return _checksum_cache.get_checksums(filelist, pn) + return _checksum_cache.get_checksums(filelist, pn, localdirsexclude) class FetchData(object): diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e484e5e37d..f982bf22bc 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -126,6 +126,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.unihash_cache = bb.cache.SimpleCache("1") self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {}) + self.localdirsexclude = (data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split() def init_rundepcheck(self, data): self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None @@ -222,9 +223,9 @@ class SignatureGeneratorBasic(SignatureGenerator): if task in dataCache.file_checksums[fn]: if self.checksum_cache: - checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) + checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude) else: - checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) + checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude) for (f,cs) in checksums: self.file_checksum_values[tid].append((f,cs)) -- cgit v1.2.3-54-g00ecf