diff options
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst | 4 | ||||
-rw-r--r-- | bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/data.py | 12 | ||||
-rw-r--r-- | bitbake/lib/bb/data_smart.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 64 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf | 2 |
6 files changed, 52 insertions, 34 deletions
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst index 7b37f6615a..f99001bc76 100644 --- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst +++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst | |||
@@ -477,7 +477,7 @@ changes because it should not affect the output for target packages. The | |||
477 | simplistic approach for excluding the working directory is to set it to | 477 | simplistic approach for excluding the working directory is to set it to |
478 | some fixed value and create the checksum for the "run" script. BitBake | 478 | some fixed value and create the checksum for the "run" script. BitBake |
479 | goes one step better and uses the | 479 | goes one step better and uses the |
480 | :term:`BB_HASHBASE_WHITELIST` variable | 480 | :term:`BB_BASEHASH_IGNORE_VARS` variable |
481 | to define a list of variables that should never be included when | 481 | to define a list of variables that should never be included when |
482 | generating the signatures. | 482 | generating the signatures. |
483 | 483 | ||
@@ -538,7 +538,7 @@ in a list of global variable dependency excludes - variables never | |||
538 | included in any checksum. This example uses variables from OpenEmbedded | 538 | included in any checksum. This example uses variables from OpenEmbedded |
539 | to help illustrate the concept:: | 539 | to help illustrate the concept:: |
540 | 540 | ||
541 | BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ | 541 | BB_BASEHASH_IGNORE_VARS ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ |
542 | SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ | 542 | SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ |
543 | USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ | 543 | USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ |
544 | PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ | 544 | PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ |
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 9d1828c3ba..bdd4127a48 100644 --- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | |||
@@ -337,7 +337,7 @@ overview of their function and contents. | |||
337 | 337 | ||
338 | For example usage, see :term:`BB_GIT_SHALLOW`. | 338 | For example usage, see :term:`BB_GIT_SHALLOW`. |
339 | 339 | ||
340 | :term:`BB_HASHBASE_WHITELIST` | 340 | :term:`BB_BASEHASH_IGNORE_VARS` |
341 | Lists variables that are excluded from checksum and dependency data. | 341 | Lists variables that are excluded from checksum and dependency data. |
342 | Variables that are excluded can therefore change without affecting | 342 | Variables that are excluded can therefore change without affecting |
343 | the checksum mechanism. A common example would be the variable for | 343 | the checksum mechanism. A common example would be the variable for |
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index ee5557abfa..5d02bab99b 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py | |||
@@ -369,7 +369,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): | |||
369 | #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) | 369 | #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) |
370 | #d.setVarFlag(key, "vardeps", deps) | 370 | #d.setVarFlag(key, "vardeps", deps) |
371 | 371 | ||
372 | def generate_dependencies(d, whitelist): | 372 | def generate_dependencies(d, ignored_vars): |
373 | 373 | ||
374 | keys = set(key for key in d if not key.startswith("__")) | 374 | keys = set(key for key in d if not key.startswith("__")) |
375 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) | 375 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) |
@@ -384,7 +384,7 @@ def generate_dependencies(d, whitelist): | |||
384 | newdeps = deps[task] | 384 | newdeps = deps[task] |
385 | seen = set() | 385 | seen = set() |
386 | while newdeps: | 386 | while newdeps: |
387 | nextdeps = newdeps - whitelist | 387 | nextdeps = newdeps - ignored_vars |
388 | seen |= nextdeps | 388 | seen |= nextdeps |
389 | newdeps = set() | 389 | newdeps = set() |
390 | for dep in nextdeps: | 390 | for dep in nextdeps: |
@@ -395,7 +395,7 @@ def generate_dependencies(d, whitelist): | |||
395 | #print "For %s: %s" % (task, str(deps[task])) | 395 | #print "For %s: %s" % (task, str(deps[task])) |
396 | return tasklist, deps, values | 396 | return tasklist, deps, values |
397 | 397 | ||
398 | def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): | 398 | def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): |
399 | taskdeps = {} | 399 | taskdeps = {} |
400 | basehash = {} | 400 | basehash = {} |
401 | 401 | ||
@@ -408,7 +408,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): | |||
408 | else: | 408 | else: |
409 | data = [data] | 409 | data = [data] |
410 | 410 | ||
411 | gendeps[task] -= whitelist | 411 | gendeps[task] -= ignored_vars |
412 | newdeps = gendeps[task] | 412 | newdeps = gendeps[task] |
413 | seen = set() | 413 | seen = set() |
414 | while newdeps: | 414 | while newdeps: |
@@ -416,9 +416,9 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn): | |||
416 | seen |= nextdeps | 416 | seen |= nextdeps |
417 | newdeps = set() | 417 | newdeps = set() |
418 | for dep in nextdeps: | 418 | for dep in nextdeps: |
419 | if dep in whitelist: | 419 | if dep in ignored_vars: |
420 | continue | 420 | continue |
421 | gendeps[dep] -= whitelist | 421 | gendeps[dep] -= ignored_vars |
422 | newdeps |= gendeps[dep] | 422 | newdeps |= gendeps[dep] |
423 | newdeps -= seen | 423 | newdeps -= seen |
424 | 424 | ||
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 01604ec36c..7cf17b0057 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py | |||
@@ -34,6 +34,8 @@ __whitespace_split__ = re.compile(r'(\s)') | |||
34 | __override_regexp__ = re.compile(r'[a-z0-9]+') | 34 | __override_regexp__ = re.compile(r'[a-z0-9]+') |
35 | 35 | ||
36 | bitbake_renamed_vars = { | 36 | bitbake_renamed_vars = { |
37 | "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", | ||
38 | "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", | ||
37 | } | 39 | } |
38 | 40 | ||
39 | def infer_caller_details(loginfo, parent = False, varval = True): | 41 | def infer_caller_details(loginfo, parent = False, varval = True): |
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e0ec736dff..d0988e0e40 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -172,8 +172,8 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
172 | self.gendeps = {} | 172 | self.gendeps = {} |
173 | self.lookupcache = {} | 173 | self.lookupcache = {} |
174 | self.setscenetasks = set() | 174 | self.setscenetasks = set() |
175 | self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) | 175 | self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) |
176 | self.taskwhitelist = None | 176 | self.taskhash_ignore_tasks = None |
177 | self.init_rundepcheck(data) | 177 | self.init_rundepcheck(data) |
178 | checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") | 178 | checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") |
179 | if checksum_cache_file: | 179 | if checksum_cache_file: |
@@ -188,18 +188,18 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
188 | self.tidtopn = {} | 188 | self.tidtopn = {} |
189 | 189 | ||
190 | def init_rundepcheck(self, data): | 190 | def init_rundepcheck(self, data): |
191 | self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None | 191 | self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None |
192 | if self.taskwhitelist: | 192 | if self.taskhash_ignore_tasks: |
193 | self.twl = re.compile(self.taskwhitelist) | 193 | self.twl = re.compile(self.taskhash_ignore_tasks) |
194 | else: | 194 | else: |
195 | self.twl = None | 195 | self.twl = None |
196 | 196 | ||
197 | def _build_data(self, fn, d): | 197 | def _build_data(self, fn, d): |
198 | 198 | ||
199 | ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') | 199 | ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') |
200 | tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) | 200 | tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars) |
201 | 201 | ||
202 | taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) | 202 | taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn) |
203 | 203 | ||
204 | for task in tasklist: | 204 | for task in tasklist: |
205 | tid = fn + ":" + task | 205 | tid = fn + ":" + task |
@@ -253,7 +253,8 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
253 | 253 | ||
254 | def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): | 254 | def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): |
255 | # Return True if we should keep the dependency, False to drop it | 255 | # Return True if we should keep the dependency, False to drop it |
256 | # We only manipulate the dependencies for packages not in the whitelist | 256 | # We only manipulate the dependencies for packages not in the ignore |
257 | # list | ||
257 | if self.twl and not self.twl.search(recipename): | 258 | if self.twl and not self.twl.search(recipename): |
258 | # then process the actual dependencies | 259 | # then process the actual dependencies |
259 | if self.twl.search(depname): | 260 | if self.twl.search(depname): |
@@ -372,15 +373,15 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
372 | 373 | ||
373 | data = {} | 374 | data = {} |
374 | data['task'] = task | 375 | data['task'] = task |
375 | data['basewhitelist'] = self.basewhitelist | 376 | data['basehash_ignore_vars'] = self.basehash_ignore_vars |
376 | data['taskwhitelist'] = self.taskwhitelist | 377 | data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks |
377 | data['taskdeps'] = self.taskdeps[fn][task] | 378 | data['taskdeps'] = self.taskdeps[fn][task] |
378 | data['basehash'] = self.basehash[tid] | 379 | data['basehash'] = self.basehash[tid] |
379 | data['gendeps'] = {} | 380 | data['gendeps'] = {} |
380 | data['varvals'] = {} | 381 | data['varvals'] = {} |
381 | data['varvals'][task] = self.lookupcache[fn][task] | 382 | data['varvals'][task] = self.lookupcache[fn][task] |
382 | for dep in self.taskdeps[fn][task]: | 383 | for dep in self.taskdeps[fn][task]: |
383 | if dep in self.basewhitelist: | 384 | if dep in self.basehash_ignore_vars: |
384 | continue | 385 | continue |
385 | data['gendeps'][dep] = self.gendeps[fn][dep] | 386 | data['gendeps'][dep] = self.gendeps[fn][dep] |
386 | data['varvals'][dep] = self.lookupcache[fn][dep] | 387 | data['varvals'][dep] = self.lookupcache[fn][dep] |
@@ -794,6 +795,16 @@ def clean_basepaths_list(a): | |||
794 | b.append(clean_basepath(x)) | 795 | b.append(clean_basepath(x)) |
795 | return b | 796 | return b |
796 | 797 | ||
798 | # Handled renamed fields | ||
799 | def handle_renames(data): | ||
800 | if 'basewhitelist' in data: | ||
801 | data['basehash_ignore_vars'] = data['basewhitelist'] | ||
802 | del data['basewhitelist'] | ||
803 | if 'taskwhitelist' in data: | ||
804 | data['taskhash_ignore_tasks'] = data['taskwhitelist'] | ||
805 | del data['taskwhitelist'] | ||
806 | |||
807 | |||
797 | def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): | 808 | def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): |
798 | output = [] | 809 | output = [] |
799 | 810 | ||
@@ -819,13 +830,16 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): | |||
819 | with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: | 830 | with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: |
820 | b_data = json.load(f, object_hook=SetDecoder) | 831 | b_data = json.load(f, object_hook=SetDecoder) |
821 | 832 | ||
822 | def dict_diff(a, b, whitelist=set()): | 833 | for data in [a_data, b_data]: |
834 | handle_renames(data) | ||
835 | |||
836 | def dict_diff(a, b, ignored_vars=set()): | ||
823 | sa = set(a.keys()) | 837 | sa = set(a.keys()) |
824 | sb = set(b.keys()) | 838 | sb = set(b.keys()) |
825 | common = sa & sb | 839 | common = sa & sb |
826 | changed = set() | 840 | changed = set() |
827 | for i in common: | 841 | for i in common: |
828 | if a[i] != b[i] and i not in whitelist: | 842 | if a[i] != b[i] and i not in ignored_vars: |
829 | changed.add(i) | 843 | changed.add(i) |
830 | added = sb - sa | 844 | added = sb - sa |
831 | removed = sa - sb | 845 | removed = sa - sb |
@@ -864,15 +878,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): | |||
864 | removed = [x[0] for x in removed] | 878 | removed = [x[0] for x in removed] |
865 | return changed, added, removed | 879 | return changed, added, removed |
866 | 880 | ||
867 | if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: | 881 | if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']: |
868 | output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) | 882 | output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars'])) |
869 | if a_data['basewhitelist'] and b_data['basewhitelist']: | 883 | if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']: |
870 | output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) | 884 | output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars'])) |
871 | 885 | ||
872 | if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: | 886 | if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']: |
873 | output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) | 887 | output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks'])) |
874 | if a_data['taskwhitelist'] and b_data['taskwhitelist']: | 888 | if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']: |
875 | output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) | 889 | output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks'])) |
876 | 890 | ||
877 | if a_data['taskdeps'] != b_data['taskdeps']: | 891 | if a_data['taskdeps'] != b_data['taskdeps']: |
878 | output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) | 892 | output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) |
@@ -880,7 +894,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): | |||
880 | if a_data['basehash'] != b_data['basehash'] and not collapsed: | 894 | if a_data['basehash'] != b_data['basehash'] and not collapsed: |
881 | output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) | 895 | output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) |
882 | 896 | ||
883 | changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) | 897 | changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars']) |
884 | if changed: | 898 | if changed: |
885 | for dep in sorted(changed): | 899 | for dep in sorted(changed): |
886 | output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) | 900 | output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) |
@@ -1054,9 +1068,11 @@ def dump_sigfile(a): | |||
1054 | with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: | 1068 | with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: |
1055 | a_data = json.load(f, object_hook=SetDecoder) | 1069 | a_data = json.load(f, object_hook=SetDecoder) |
1056 | 1070 | ||
1057 | output.append("basewhitelist: %s" % (sorted(a_data['basewhitelist']))) | 1071 | handle_renames(a_data) |
1072 | |||
1073 | output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars']))) | ||
1058 | 1074 | ||
1059 | output.append("taskwhitelist: %s" % (sorted(a_data['taskwhitelist'] or []))) | 1075 | output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or []))) |
1060 | 1076 | ||
1061 | output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) | 1077 | output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) |
1062 | 1078 | ||
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 2645c0e985..05d7fd07dd 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf | |||
@@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}" | |||
12 | T = "${TMPDIR}/workdir/${PN}/temp" | 12 | T = "${TMPDIR}/workdir/${PN}/temp" |
13 | BB_NUMBER_THREADS = "4" | 13 | BB_NUMBER_THREADS = "4" |
14 | 14 | ||
15 | BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" | 15 | BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" |
16 | 16 | ||
17 | include conf/multiconfig/${BB_CURRENT_MC}.conf | 17 | include conf/multiconfig/${BB_CURRENT_MC}.conf |