summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/data.py12
-rw-r--r--bitbake/lib/bb/data_smart.py2
-rw-r--r--bitbake/lib/bb/siggen.py64
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf2
4 files changed, 49 insertions, 31 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index ee5557abfa..5d02bab99b 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -369,7 +369,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
369 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) 369 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
370 #d.setVarFlag(key, "vardeps", deps) 370 #d.setVarFlag(key, "vardeps", deps)
371 371
372def generate_dependencies(d, whitelist): 372def generate_dependencies(d, ignored_vars):
373 373
374 keys = set(key for key in d if not key.startswith("__")) 374 keys = set(key for key in d if not key.startswith("__"))
375 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) 375 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -384,7 +384,7 @@ def generate_dependencies(d, whitelist):
384 newdeps = deps[task] 384 newdeps = deps[task]
385 seen = set() 385 seen = set()
386 while newdeps: 386 while newdeps:
387 nextdeps = newdeps - whitelist 387 nextdeps = newdeps - ignored_vars
388 seen |= nextdeps 388 seen |= nextdeps
389 newdeps = set() 389 newdeps = set()
390 for dep in nextdeps: 390 for dep in nextdeps:
@@ -395,7 +395,7 @@ def generate_dependencies(d, whitelist):
395 #print "For %s: %s" % (task, str(deps[task])) 395 #print "For %s: %s" % (task, str(deps[task]))
396 return tasklist, deps, values 396 return tasklist, deps, values
397 397
398def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): 398def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
399 taskdeps = {} 399 taskdeps = {}
400 basehash = {} 400 basehash = {}
401 401
@@ -408,7 +408,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
408 else: 408 else:
409 data = [data] 409 data = [data]
410 410
411 gendeps[task] -= whitelist 411 gendeps[task] -= ignored_vars
412 newdeps = gendeps[task] 412 newdeps = gendeps[task]
413 seen = set() 413 seen = set()
414 while newdeps: 414 while newdeps:
@@ -416,9 +416,9 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
416 seen |= nextdeps 416 seen |= nextdeps
417 newdeps = set() 417 newdeps = set()
418 for dep in nextdeps: 418 for dep in nextdeps:
419 if dep in whitelist: 419 if dep in ignored_vars:
420 continue 420 continue
421 gendeps[dep] -= whitelist 421 gendeps[dep] -= ignored_vars
422 newdeps |= gendeps[dep] 422 newdeps |= gendeps[dep]
423 newdeps -= seen 423 newdeps -= seen
424 424
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 01604ec36c..7cf17b0057 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -34,6 +34,8 @@ __whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+') 34__override_regexp__ = re.compile(r'[a-z0-9]+')
35 35
36bitbake_renamed_vars = { 36bitbake_renamed_vars = {
37 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
38 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
37} 39}
38 40
39def infer_caller_details(loginfo, parent = False, varval = True): 41def infer_caller_details(loginfo, parent = False, varval = True):
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index e0ec736dff..d0988e0e40 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -172,8 +172,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
172 self.gendeps = {} 172 self.gendeps = {}
173 self.lookupcache = {} 173 self.lookupcache = {}
174 self.setscenetasks = set() 174 self.setscenetasks = set()
175 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) 175 self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split())
176 self.taskwhitelist = None 176 self.taskhash_ignore_tasks = None
177 self.init_rundepcheck(data) 177 self.init_rundepcheck(data)
178 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") 178 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
179 if checksum_cache_file: 179 if checksum_cache_file:
@@ -188,18 +188,18 @@ class SignatureGeneratorBasic(SignatureGenerator):
188 self.tidtopn = {} 188 self.tidtopn = {}
189 189
190 def init_rundepcheck(self, data): 190 def init_rundepcheck(self, data):
191 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None 191 self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None
192 if self.taskwhitelist: 192 if self.taskhash_ignore_tasks:
193 self.twl = re.compile(self.taskwhitelist) 193 self.twl = re.compile(self.taskhash_ignore_tasks)
194 else: 194 else:
195 self.twl = None 195 self.twl = None
196 196
197 def _build_data(self, fn, d): 197 def _build_data(self, fn, d):
198 198
199 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') 199 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
200 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) 200 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars)
201 201
202 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) 202 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn)
203 203
204 for task in tasklist: 204 for task in tasklist:
205 tid = fn + ":" + task 205 tid = fn + ":" + task
@@ -253,7 +253,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
253 253
254 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): 254 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
255 # Return True if we should keep the dependency, False to drop it 255 # Return True if we should keep the dependency, False to drop it
256 # We only manipulate the dependencies for packages not in the whitelist 256 # We only manipulate the dependencies for packages not in the ignore
257 # list
257 if self.twl and not self.twl.search(recipename): 258 if self.twl and not self.twl.search(recipename):
258 # then process the actual dependencies 259 # then process the actual dependencies
259 if self.twl.search(depname): 260 if self.twl.search(depname):
@@ -372,15 +373,15 @@ class SignatureGeneratorBasic(SignatureGenerator):
372 373
373 data = {} 374 data = {}
374 data['task'] = task 375 data['task'] = task
375 data['basewhitelist'] = self.basewhitelist 376 data['basehash_ignore_vars'] = self.basehash_ignore_vars
376 data['taskwhitelist'] = self.taskwhitelist 377 data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks
377 data['taskdeps'] = self.taskdeps[fn][task] 378 data['taskdeps'] = self.taskdeps[fn][task]
378 data['basehash'] = self.basehash[tid] 379 data['basehash'] = self.basehash[tid]
379 data['gendeps'] = {} 380 data['gendeps'] = {}
380 data['varvals'] = {} 381 data['varvals'] = {}
381 data['varvals'][task] = self.lookupcache[fn][task] 382 data['varvals'][task] = self.lookupcache[fn][task]
382 for dep in self.taskdeps[fn][task]: 383 for dep in self.taskdeps[fn][task]:
383 if dep in self.basewhitelist: 384 if dep in self.basehash_ignore_vars:
384 continue 385 continue
385 data['gendeps'][dep] = self.gendeps[fn][dep] 386 data['gendeps'][dep] = self.gendeps[fn][dep]
386 data['varvals'][dep] = self.lookupcache[fn][dep] 387 data['varvals'][dep] = self.lookupcache[fn][dep]
@@ -794,6 +795,16 @@ def clean_basepaths_list(a):
794 b.append(clean_basepath(x)) 795 b.append(clean_basepath(x))
795 return b 796 return b
796 797
798# Handled renamed fields
799def handle_renames(data):
800 if 'basewhitelist' in data:
801 data['basehash_ignore_vars'] = data['basewhitelist']
802 del data['basewhitelist']
803 if 'taskwhitelist' in data:
804 data['taskhash_ignore_tasks'] = data['taskwhitelist']
805 del data['taskwhitelist']
806
807
797def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): 808def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
798 output = [] 809 output = []
799 810
@@ -819,13 +830,16 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
819 with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: 830 with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
820 b_data = json.load(f, object_hook=SetDecoder) 831 b_data = json.load(f, object_hook=SetDecoder)
821 832
822 def dict_diff(a, b, whitelist=set()): 833 for data in [a_data, b_data]:
834 handle_renames(data)
835
836 def dict_diff(a, b, ignored_vars=set()):
823 sa = set(a.keys()) 837 sa = set(a.keys())
824 sb = set(b.keys()) 838 sb = set(b.keys())
825 common = sa & sb 839 common = sa & sb
826 changed = set() 840 changed = set()
827 for i in common: 841 for i in common:
828 if a[i] != b[i] and i not in whitelist: 842 if a[i] != b[i] and i not in ignored_vars:
829 changed.add(i) 843 changed.add(i)
830 added = sb - sa 844 added = sb - sa
831 removed = sa - sb 845 removed = sa - sb
@@ -864,15 +878,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
864 removed = [x[0] for x in removed] 878 removed = [x[0] for x in removed]
865 return changed, added, removed 879 return changed, added, removed
866 880
867 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: 881 if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']:
868 output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) 882 output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars']))
869 if a_data['basewhitelist'] and b_data['basewhitelist']: 883 if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']:
870 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) 884 output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars']))
871 885
872 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: 886 if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']:
873 output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) 887 output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks']))
874 if a_data['taskwhitelist'] and b_data['taskwhitelist']: 888 if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']:
875 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) 889 output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks']))
876 890
877 if a_data['taskdeps'] != b_data['taskdeps']: 891 if a_data['taskdeps'] != b_data['taskdeps']:
878 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) 892 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
@@ -880,7 +894,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
880 if a_data['basehash'] != b_data['basehash'] and not collapsed: 894 if a_data['basehash'] != b_data['basehash'] and not collapsed:
881 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) 895 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
882 896
883 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) 897 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars'])
884 if changed: 898 if changed:
885 for dep in sorted(changed): 899 for dep in sorted(changed):
886 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) 900 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
@@ -1054,9 +1068,11 @@ def dump_sigfile(a):
1054 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: 1068 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
1055 a_data = json.load(f, object_hook=SetDecoder) 1069 a_data = json.load(f, object_hook=SetDecoder)
1056 1070
1057 output.append("basewhitelist: %s" % (sorted(a_data['basewhitelist']))) 1071 handle_renames(a_data)
1072
1073 output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars'])))
1058 1074
1059 output.append("taskwhitelist: %s" % (sorted(a_data['taskwhitelist'] or []))) 1075 output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or [])))
1060 1076
1061 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) 1077 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
1062 1078
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 2645c0e985..05d7fd07dd 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
12T = "${TMPDIR}/workdir/${PN}/temp" 12T = "${TMPDIR}/workdir/${PN}/temp"
13BB_NUMBER_THREADS = "4" 13BB_NUMBER_THREADS = "4"
14 14
15BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" 15BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK"
16 16
17include conf/multiconfig/${BB_CURRENT_MC}.conf 17include conf/multiconfig/${BB_CURRENT_MC}.conf