summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-12-03 12:10:05 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-12-03 17:45:53 +0000
commiteba506b301b3be7953a64b486c231b0e7c7c1de3 (patch)
treea22806e194598e9c16d78dfc92daadd0017d0be3 /bitbake/lib/bb
parentb4b1c90bcfe4b3baddabd5bfcea9a264be223908 (diff)
downloadpoky-eba506b301b3be7953a64b486c231b0e7c7c1de3.tar.gz
bitbake: codeparser/data_smart: Optimise parsing speed
The previous "contains" changes caused a ~3% parsing speed impact. Looking at the cause of those changes was interesting: * Use of defaultdict was slower than just checking for missing entries and setting them when needed. * Even the "import collections" adversely affects parsing speed * There was a missing intern function for the contains cache data * Setting up a log object for each variable has noticeable overhead due to the changes in the code paths uses, we can avoid this. * We can call getVarFlag on "_content" directly within VariableParse for a noticeable speed gain since its a seriously hot code path. This patch therefore tweaks the code based on the above observations to get some of the speed back. (Bitbake rev: fca802187a2a30686a8a07d2b6b16a3e5716e293) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/codeparser.py9
-rw-r--r--bitbake/lib/bb/data_smart.py10
2 files changed, 12 insertions, 7 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 6e34eff999..62b6cf9e3a 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -1,7 +1,6 @@
1import ast 1import ast
2import codegen 2import codegen
3import logging 3import logging
4import collections
5import os.path 4import os.path
6import bb.utils, bb.data 5import bb.utils, bb.data
7from itertools import chain 6from itertools import chain
@@ -65,6 +64,8 @@ class CodeParserCache(MultiProcessCache):
65 for h in data[0]: 64 for h in data[0]:
66 data[0][h]["refs"] = self.internSet(data[0][h]["refs"]) 65 data[0][h]["refs"] = self.internSet(data[0][h]["refs"])
67 data[0][h]["execs"] = self.internSet(data[0][h]["execs"]) 66 data[0][h]["execs"] = self.internSet(data[0][h]["execs"])
67 for k in data[0][h]["contains"]:
68 data[0][h]["contains"][k] = self.internSet(data[0][h]["contains"][k])
68 for h in data[1]: 69 for h in data[1]:
69 data[1][h]["execs"] = self.internSet(data[1][h]["execs"]) 70 data[1][h]["execs"] = self.internSet(data[1][h]["execs"])
70 return 71 return
@@ -125,6 +126,8 @@ class PythonParser():
125 if isinstance(node.args[0], ast.Str): 126 if isinstance(node.args[0], ast.Str):
126 varname = node.args[0].s 127 varname = node.args[0].s
127 if name in self.containsfuncs and isinstance(node.args[1], ast.Str): 128 if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
129 if varname not in self.contains:
130 self.contains[varname] = set()
128 self.contains[varname].add(node.args[1].s) 131 self.contains[varname].add(node.args[1].s)
129 else: 132 else:
130 self.references.add(node.args[0].s) 133 self.references.add(node.args[0].s)
@@ -153,10 +156,10 @@ class PythonParser():
153 156
154 def __init__(self, name, log): 157 def __init__(self, name, log):
155 self.var_execs = set() 158 self.var_execs = set()
156 self.contains = collections.defaultdict(set) 159 self.contains = {}
157 self.execs = set() 160 self.execs = set()
158 self.references = set() 161 self.references = set()
159 self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log) 162 self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
160 163
161 self.unhandled_message = "in call of %s, argument '%s' is not a string literal" 164 self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
162 self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) 165 self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 9a6f767116..833d9f17a4 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -35,7 +35,6 @@ import hashlib
35import bb, bb.codeparser 35import bb, bb.codeparser
36from bb import utils 36from bb import utils
37from bb.COW import COWDictBase 37from bb.COW import COWDictBase
38import collections
39 38
40logger = logging.getLogger("BitBake.Data") 39logger = logging.getLogger("BitBake.Data")
41 40
@@ -89,7 +88,7 @@ class VariableParse:
89 88
90 self.references = set() 89 self.references = set()
91 self.execs = set() 90 self.execs = set()
92 self.contains = collections.defaultdict(set) 91 self.contains = {}
93 92
94 def var_sub(self, match): 93 def var_sub(self, match):
95 key = match.group()[2:-1] 94 key = match.group()[2:-1]
@@ -100,7 +99,7 @@ class VariableParse:
100 varparse = self.d.expand_cache[key] 99 varparse = self.d.expand_cache[key]
101 var = varparse.value 100 var = varparse.value
102 else: 101 else:
103 var = self.d.getVar(key, True) 102 var = self.d.getVarFlag(key, "_content", True)
104 self.references.add(key) 103 self.references.add(key)
105 if var is not None: 104 if var is not None:
106 return var 105 return var
@@ -123,7 +122,10 @@ class VariableParse:
123 self.execs |= parser.execs 122 self.execs |= parser.execs
124 123
125 for k in parser.contains: 124 for k in parser.contains:
126 self.contains[k].update(parser.contains[k]) 125 if k not in self.contains:
126 self.contains[k] = parser.contains[k]
127 else:
128 self.contains[k].update(parser.contains[k])
127 value = utils.better_eval(codeobj, DataContext(self.d)) 129 value = utils.better_eval(codeobj, DataContext(self.d))
128 return str(value) 130 return str(value)
129 131