From 8a16bf5321a51f440b9b5dcc29bc498d1eb0bf28 Mon Sep 17 00:00:00 2001 From: Scott Murray Date: Sun, 13 Feb 2022 16:54:25 -0500 Subject: bitbake: bitbake: Rename basehas and taskhash filtering variables In line with the inclusive language migration defined at: https://wiki.yoctoproject.org/wiki/Inclusive_language rename: BB_HASHBASE_WHITELIST -> BB_BASEHASH_IGNORE_VARS BB_HASHTASK_WHITELIST -> BB_TASKHASH_IGNORE_TASKS the derived code variables basewhitelist and taskwhitelist have been renamed to basehash_ignore_vars and taskhash_ignore_tasks, respectively. [RP: Added compatibility mapping code and tweaked naming] (Bitbake rev: efaafc9ec2e8c0475e3fb27e877a1c0a5532a0e5) Signed-off-by: Scott Murray Signed-off-by: Richard Purdie --- .../bitbake-user-manual-execution.rst | 4 +- .../bitbake-user-manual-ref-variables.rst | 2 +- bitbake/lib/bb/data.py | 12 ++-- bitbake/lib/bb/data_smart.py | 2 + bitbake/lib/bb/siggen.py | 64 ++++++++++++++-------- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 2 +- 6 files changed, 52 insertions(+), 34 deletions(-) diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst index 7b37f6615a..f99001bc76 100644 --- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst +++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst @@ -477,7 +477,7 @@ changes because it should not affect the output for target packages. The simplistic approach for excluding the working directory is to set it to some fixed value and create the checksum for the "run" script. BitBake goes one step better and uses the -:term:`BB_HASHBASE_WHITELIST` variable +:term:`BB_BASEHASH_IGNORE_VARS` variable to define a list of variables that should never be included when generating the signatures. @@ -538,7 +538,7 @@ in a list of global variable dependency excludes - variables never included in any checksum. This example uses variables from OpenEmbedded to help illustrate the concept:: - BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ + BB_BASEHASH_IGNORE_VARS ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 9d1828c3ba..bdd4127a48 100644 --- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -337,7 +337,7 @@ overview of their function and contents. For example usage, see :term:`BB_GIT_SHALLOW`. - :term:`BB_HASHBASE_WHITELIST` + :term:`BB_BASEHASH_IGNORE_VARS` Lists variables that are excluded from checksum and dependency data. Variables that are excluded can therefore change without affecting the checksum mechanism. A common example would be the variable for diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index ee5557abfa..5d02bab99b 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py @@ -369,7 +369,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) #d.setVarFlag(key, "vardeps", deps) -def generate_dependencies(d, whitelist): +def generate_dependencies(d, ignored_vars): keys = set(key for key in d if not key.startswith("__")) shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) @@ -384,7 +384,7 @@ def generate_dependencies(d, whitelist): newdeps = deps[task] seen = set() while newdeps: - nextdeps = newdeps - whitelist + nextdeps = newdeps - ignored_vars seen |= nextdeps newdeps = set() for dep in nextdeps: @@ -395,7 +395,7 @@ def generate_dependencies(d, whitelist): #print "For %s: %s" % (task, str(deps[task])) return tasklist, deps, values -def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): +def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): taskdeps = {} basehash = {} @@ -408,7 +408,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): else: data = [data] - gendeps[task] -= whitelist + gendeps[task] -= ignored_vars newdeps = gendeps[task] seen = set() while newdeps: @@ -416,9 +416,9 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): seen |= nextdeps newdeps = set() for dep in nextdeps: - if dep in whitelist: + if dep in ignored_vars: continue - gendeps[dep] -= whitelist + gendeps[dep] -= ignored_vars newdeps |= gendeps[dep] newdeps -= seen diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 01604ec36c..7cf17b0057 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -34,6 +34,8 @@ __whitespace_split__ = re.compile(r'(\s)') __override_regexp__ = re.compile(r'[a-z0-9]+') bitbake_renamed_vars = { + "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", + "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", } def infer_caller_details(loginfo, parent = False, varval = True): diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e0ec736dff..d0988e0e40 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -172,8 +172,8 @@ class SignatureGeneratorBasic(SignatureGenerator): self.gendeps = {} self.lookupcache = {} self.setscenetasks = set() - self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) - self.taskwhitelist = None + self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) + self.taskhash_ignore_tasks = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: @@ -188,18 +188,18 @@ class SignatureGeneratorBasic(SignatureGenerator): self.tidtopn = {} def init_rundepcheck(self, data): - self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None - if self.taskwhitelist: - self.twl = re.compile(self.taskwhitelist) + self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None + if self.taskhash_ignore_tasks: + self.twl = re.compile(self.taskhash_ignore_tasks) else: self.twl = None def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars) - taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) + taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn) for task in tasklist: tid = fn + ":" + task @@ -253,7 +253,8 @@ class SignatureGeneratorBasic(SignatureGenerator): def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): # Return True if we should keep the dependency, False to drop it - # We only manipulate the dependencies for packages not in the whitelist + # We only manipulate the dependencies for packages not in the ignore + # list if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): @@ -372,15 +373,15 @@ class SignatureGeneratorBasic(SignatureGenerator): data = {} data['task'] = task - data['basewhitelist'] = self.basewhitelist - data['taskwhitelist'] = self.taskwhitelist + data['basehash_ignore_vars'] = self.basehash_ignore_vars + data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: - if dep in self.basewhitelist: + if dep in self.basehash_ignore_vars: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] @@ -794,6 +795,16 @@ def clean_basepaths_list(a): b.append(clean_basepath(x)) return b +# Handled renamed fields +def handle_renames(data): + if 'basewhitelist' in data: + data['basehash_ignore_vars'] = data['basewhitelist'] + del data['basewhitelist'] + if 'taskwhitelist' in data: + data['taskhash_ignore_tasks'] = data['taskwhitelist'] + del data['taskwhitelist'] + + def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): output = [] @@ -819,13 +830,16 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: b_data = json.load(f, object_hook=SetDecoder) - def dict_diff(a, b, whitelist=set()): + for data in [a_data, b_data]: + handle_renames(data) + + def dict_diff(a, b, ignored_vars=set()): sa = set(a.keys()) sb = set(b.keys()) common = sa & sb changed = set() for i in common: - if a[i] != b[i] and i not in whitelist: + if a[i] != b[i] and i not in ignored_vars: changed.add(i) added = sb - sa removed = sa - sb @@ -864,15 +878,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): removed = [x[0] for x in removed] return changed, added, removed - if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: - output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) - if a_data['basewhitelist'] and b_data['basewhitelist']: - output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) + if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']: + output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars'])) + if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']: + output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars'])) - if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: - output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) - if a_data['taskwhitelist'] and b_data['taskwhitelist']: - output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) + if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']: + output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks'])) + if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']: + output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks'])) if a_data['taskdeps'] != b_data['taskdeps']: output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) @@ -880,7 +894,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): if a_data['basehash'] != b_data['basehash'] and not collapsed: output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) - changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) + changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars']) if changed: for dep in sorted(changed): output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) @@ -1054,9 +1068,11 @@ def dump_sigfile(a): with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: a_data = json.load(f, object_hook=SetDecoder) - output.append("basewhitelist: %s" % (sorted(a_data['basewhitelist']))) + handle_renames(a_data) + + output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars']))) - output.append("taskwhitelist: %s" % (sorted(a_data['taskwhitelist'] or []))) + output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or []))) output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 2645c0e985..05d7fd07dd 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}" T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" -BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" +BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" include conf/multiconfig/${BB_CURRENT_MC}.conf -- cgit v1.2.3-54-g00ecf