summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-11-22 09:54:11 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-11-23 18:27:07 +0000
commit61fac62194b3c7e1c31a9119390f4f960077dea4 (patch)
tree8df9d6b92f187b61894972dbb1851ea4c50d3dea
parentf68a10b5cf7e0ffc558b01d56bd7801a0696cf29 (diff)
downloadpoky-61fac62194b3c7e1c31a9119390f4f960077dea4.tar.gz
bitbake: data/siggen: Switch to use frozensets and optimize
Python handles frozensets a little more optimally than normal sets. Once we finish parsing, we don't edit this data so we can convert to them. To do that, we need to stop changing them so process ignore_deps earlier then we can freeze the data and keep it frozen. This has the side effect that we need to be careful to sort the data in some of the variables when calculating the hashes. Overall this does seem to show a decent parsing time speed improvement of 20-25% in a local test but this would be highly setup dependent. Also ensure the sigdata can handle exported frozenset and make it import back to them instead of sets. (Bitbake rev: 19475627c363a52da49ec144422c87448ff2a6c5) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/data.py14
-rw-r--r--bitbake/lib/bb/siggen.py6
2 files changed, 9 insertions, 11 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 3a6af325f4..430d444203 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -276,7 +276,8 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d):
276 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True) 276 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
277 deps |= parser.references 277 deps |= parser.references
278 deps = deps | (keys & parser.execs) 278 deps = deps | (keys & parser.execs)
279 return deps, value 279 deps -= ignored_vars
280 return frozenset(deps), value
280 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {} 281 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
281 vardeps = varflags.get("vardeps") 282 vardeps = varflags.get("vardeps")
282 exclusions = varflags.get("vardepsexclude", "").split() 283 exclusions = varflags.get("vardepsexclude", "").split()
@@ -359,12 +360,13 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d):
359 360
360 deps |= set((vardeps or "").split()) 361 deps |= set((vardeps or "").split())
361 deps -= set(exclusions) 362 deps -= set(exclusions)
363 deps -= ignored_vars
362 except bb.parse.SkipRecipe: 364 except bb.parse.SkipRecipe:
363 raise 365 raise
364 except Exception as e: 366 except Exception as e:
365 bb.warn("Exception during build_dependencies for %s" % key) 367 bb.warn("Exception during build_dependencies for %s" % key)
366 raise 368 raise
367 return deps, value 369 return frozenset(deps), value
368 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) 370 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
369 #d.setVarFlag(key, "vardeps", deps) 371 #d.setVarFlag(key, "vardeps", deps)
370 372
@@ -383,7 +385,7 @@ def generate_dependencies(d, ignored_vars):
383 newdeps = deps[task] 385 newdeps = deps[task]
384 seen = set() 386 seen = set()
385 while newdeps: 387 while newdeps:
386 nextdeps = newdeps - ignored_vars 388 nextdeps = newdeps
387 seen |= nextdeps 389 seen |= nextdeps
388 newdeps = set() 390 newdeps = set()
389 for dep in nextdeps: 391 for dep in nextdeps:
@@ -407,7 +409,6 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
407 else: 409 else:
408 data = [data] 410 data = [data]
409 411
410 gendeps[task] -= ignored_vars
411 newdeps = gendeps[task] 412 newdeps = gendeps[task]
412 seen = set() 413 seen = set()
413 while newdeps: 414 while newdeps:
@@ -415,9 +416,6 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
415 seen |= nextdeps 416 seen |= nextdeps
416 newdeps = set() 417 newdeps = set()
417 for dep in nextdeps: 418 for dep in nextdeps:
418 if dep in ignored_vars:
419 continue
420 gendeps[dep] -= ignored_vars
421 newdeps |= gendeps[dep] 419 newdeps |= gendeps[dep]
422 newdeps -= seen 420 newdeps -= seen
423 421
@@ -429,7 +427,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
429 data.append(str(var)) 427 data.append(str(var))
430 k = fn + ":" + task 428 k = fn + ":" + task
431 basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest() 429 basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest()
432 taskdeps[task] = alldeps 430 taskdeps[task] = frozenset(seen)
433 431
434 return taskdeps, basehash 432 return taskdeps, basehash
435 433
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 07bb529452..72b906c153 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -25,13 +25,13 @@ hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
25 25
26class SetEncoder(json.JSONEncoder): 26class SetEncoder(json.JSONEncoder):
27 def default(self, obj): 27 def default(self, obj):
28 if isinstance(obj, set): 28 if isinstance(obj, set) or isinstance(obj, frozenset):
29 return dict(_set_object=list(sorted(obj))) 29 return dict(_set_object=list(sorted(obj)))
30 return json.JSONEncoder.default(self, obj) 30 return json.JSONEncoder.default(self, obj)
31 31
32def SetDecoder(dct): 32def SetDecoder(dct):
33 if '_set_object' in dct: 33 if '_set_object' in dct:
34 return set(dct['_set_object']) 34 return frozenset(dct['_set_object'])
35 return dct 35 return dct
36 36
37def init(d): 37def init(d):
@@ -1056,7 +1056,7 @@ def calc_basehash(sigdata):
1056 basedata = '' 1056 basedata = ''
1057 1057
1058 alldeps = sigdata['taskdeps'] 1058 alldeps = sigdata['taskdeps']
1059 for dep in alldeps: 1059 for dep in sorted(alldeps):
1060 basedata = basedata + dep 1060 basedata = basedata + dep
1061 val = sigdata['varvals'][dep] 1061 val = sigdata['varvals'][dep]
1062 if val is not None: 1062 if val is not None: