From 8a16bf5321a51f440b9b5dcc29bc498d1eb0bf28 Mon Sep 17 00:00:00 2001 From: Scott Murray Date: Sun, 13 Feb 2022 16:54:25 -0500 Subject: bitbake: bitbake: Rename basehas and taskhash filtering variables In line with the inclusive language migration defined at: https://wiki.yoctoproject.org/wiki/Inclusive_language rename: BB_HASHBASE_WHITELIST -> BB_BASEHASH_IGNORE_VARS BB_HASHTASK_WHITELIST -> BB_TASKHASH_IGNORE_TASKS the derived code variables basewhitelist and taskwhitelist have been renamed to basehash_ignore_vars and taskhash_ignore_tasks, respectively. [RP: Added compatibility mapping code and tweaked naming] (Bitbake rev: efaafc9ec2e8c0475e3fb27e877a1c0a5532a0e5) Signed-off-by: Scott Murray Signed-off-by: Richard Purdie --- bitbake/lib/bb/data.py | 12 ++-- bitbake/lib/bb/data_smart.py | 2 + bitbake/lib/bb/siggen.py | 64 ++++++++++++++-------- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 2 +- 4 files changed, 49 insertions(+), 31 deletions(-) (limited to 'bitbake/lib/bb') diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index ee5557abfa..5d02bab99b 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py @@ -369,7 +369,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) #d.setVarFlag(key, "vardeps", deps) -def generate_dependencies(d, whitelist): +def generate_dependencies(d, ignored_vars): keys = set(key for key in d if not key.startswith("__")) shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) @@ -384,7 +384,7 @@ def generate_dependencies(d, whitelist): newdeps = deps[task] seen = set() while newdeps: - nextdeps = newdeps - whitelist + nextdeps = newdeps - ignored_vars seen |= nextdeps newdeps = set() for dep in nextdeps: @@ -395,7 +395,7 @@ def generate_dependencies(d, whitelist): #print "For %s: %s" % (task, str(deps[task])) return tasklist, deps, values -def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): +def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): taskdeps = {} basehash = {} @@ -408,7 +408,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): else: data = [data] - gendeps[task] -= whitelist + gendeps[task] -= ignored_vars newdeps = gendeps[task] seen = set() while newdeps: @@ -416,9 +416,9 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): seen |= nextdeps newdeps = set() for dep in nextdeps: - if dep in whitelist: + if dep in ignored_vars: continue - gendeps[dep] -= whitelist + gendeps[dep] -= ignored_vars newdeps |= gendeps[dep] newdeps -= seen diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 01604ec36c..7cf17b0057 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -34,6 +34,8 @@ __whitespace_split__ = re.compile(r'(\s)') __override_regexp__ = re.compile(r'[a-z0-9]+') bitbake_renamed_vars = { + "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", + "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", } def infer_caller_details(loginfo, parent = False, varval = True): diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e0ec736dff..d0988e0e40 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -172,8 +172,8 @@ class SignatureGeneratorBasic(SignatureGenerator): self.gendeps = {} self.lookupcache = {} self.setscenetasks = set() - self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) - self.taskwhitelist = None + self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) + self.taskhash_ignore_tasks = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: @@ -188,18 +188,18 @@ class SignatureGeneratorBasic(SignatureGenerator): self.tidtopn = {} def init_rundepcheck(self, data): - self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None - if self.taskwhitelist: - self.twl = re.compile(self.taskwhitelist) + self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None + if self.taskhash_ignore_tasks: + self.twl = re.compile(self.taskhash_ignore_tasks) else: self.twl = None def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars) - taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) + taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn) for task in tasklist: tid = fn + ":" + task @@ -253,7 +253,8 @@ class SignatureGeneratorBasic(SignatureGenerator): def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): # Return True if we should keep the dependency, False to drop it - # We only manipulate the dependencies for packages not in the whitelist + # We only manipulate the dependencies for packages not in the ignore + # list if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): @@ -372,15 +373,15 @@ class SignatureGeneratorBasic(SignatureGenerator): data = {} data['task'] = task - data['basewhitelist'] = self.basewhitelist - data['taskwhitelist'] = self.taskwhitelist + data['basehash_ignore_vars'] = self.basehash_ignore_vars + data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: - if dep in self.basewhitelist: + if dep in self.basehash_ignore_vars: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] @@ -794,6 +795,16 @@ def clean_basepaths_list(a): b.append(clean_basepath(x)) return b +# Handled renamed fields +def handle_renames(data): + if 'basewhitelist' in data: + data['basehash_ignore_vars'] = data['basewhitelist'] + del data['basewhitelist'] + if 'taskwhitelist' in data: + data['taskhash_ignore_tasks'] = data['taskwhitelist'] + del data['taskwhitelist'] + + def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): output = [] @@ -819,13 +830,16 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: b_data = json.load(f, object_hook=SetDecoder) - def dict_diff(a, b, whitelist=set()): + for data in [a_data, b_data]: + handle_renames(data) + + def dict_diff(a, b, ignored_vars=set()): sa = set(a.keys()) sb = set(b.keys()) common = sa & sb changed = set() for i in common: - if a[i] != b[i] and i not in whitelist: + if a[i] != b[i] and i not in ignored_vars: changed.add(i) added = sb - sa removed = sa - sb @@ -864,15 +878,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): removed = [x[0] for x in removed] return changed, added, removed - if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: - output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) - if a_data['basewhitelist'] and b_data['basewhitelist']: - output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) + if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']: + output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars'])) + if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']: + output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars'])) - if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: - output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) - if a_data['taskwhitelist'] and b_data['taskwhitelist']: - output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) + if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']: + output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks'])) + if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']: + output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks'])) if a_data['taskdeps'] != b_data['taskdeps']: output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) @@ -880,7 +894,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): if a_data['basehash'] != b_data['basehash'] and not collapsed: output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) - changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) + changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars']) if changed: for dep in sorted(changed): output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) @@ -1054,9 +1068,11 @@ def dump_sigfile(a): with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: a_data = json.load(f, object_hook=SetDecoder) - output.append("basewhitelist: %s" % (sorted(a_data['basewhitelist']))) + handle_renames(a_data) + + output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars']))) - output.append("taskwhitelist: %s" % (sorted(a_data['taskwhitelist'] or []))) + output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or []))) output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 2645c0e985..05d7fd07dd 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}" T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" -BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" +BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" include conf/multiconfig/${BB_CURRENT_MC}.conf -- cgit v1.2.3-54-g00ecf