summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/codeparser.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2011-05-27 16:14:48 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-05-27 17:55:51 +0100
commitfd005661443b327342b9542ab42177960e1a91a9 (patch)
treeb7d3cd3c15fc8076b82b9376360ffb22b8561e27 /bitbake/lib/bb/codeparser.py
parentc373727bd0551651621cb83ee934f84e58aa9ee1 (diff)
downloadpoky-fd005661443b327342b9542ab42177960e1a91a9.tar.gz
bitbake/codeparser: Improve cache handling
The current codeparser cache handling hurts performance badly even over a couple of cores and certainly on many core systems, it can spent huge amounts of time in the codeparser cache save functions. This patch reworks the cache handling so that each parsing thread saves out its own "differences" file compared to any existing core cache and then the main bitbake thread picks these up and merges things back together. This was tested on systems with small and large numbers of cores and was found to perform orders of magnitude better in all cases despite the more complex code. (Bitbake rev: 9f27563d66523f5af1028f173d53ee75e0877d46) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/codeparser.py')
-rw-r--r--bitbake/lib/bb/codeparser.py80
1 files changed, 70 insertions, 10 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index bb26167481..936b742614 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -70,7 +70,51 @@ def parser_cache_save(d):
70 if not cachefile: 70 if not cachefile:
71 return 71 return
72 72
73 lf = bb.utils.lockfile(cachefile + ".lock") 73 glf = bb.utils.lockfile(cachefile + ".lock", shared=True)
74
75 i = os.getpid()
76 lf = None
77 while not lf:
78 shellcache = {}
79 pythoncache = {}
80
81 lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False)
82 if not lf or os.path.exists(cachefile + "-" + str(i)):
83 if lf:
84 bb.utils.unlockfile(lf)
85 lf = None
86 i = i + 1
87 continue
88
89 try:
90 p = pickle.Unpickler(file(cachefile, "rb"))
91 data, version = p.load()
92 except (IOError, EOFError):
93 data, version = None, None
94
95 if version != PARSERCACHE_VERSION:
96 shellcache = shellparsecache
97 pythoncache = pythonparsecache
98 else:
99 for h in pythonparsecache:
100 if h not in data[0]:
101 pythoncache[h] = pythonparsecache[h]
102 for h in shellparsecache:
103 if h not in data[1]:
104 shellcache[h] = shellparsecache[h]
105
106 p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
107 p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
108
109 bb.utils.unlockfile(lf)
110 bb.utils.unlockfile(glf)
111
112def parser_cache_savemerge(d):
113 cachefile = parser_cachefile(d)
114 if not cachefile:
115 return
116
117 glf = bb.utils.lockfile(cachefile + ".lock")
74 118
75 try: 119 try:
76 p = pickle.Unpickler(file(cachefile, "rb")) 120 p = pickle.Unpickler(file(cachefile, "rb"))
@@ -78,17 +122,33 @@ def parser_cache_save(d):
78 except (IOError, EOFError): 122 except (IOError, EOFError):
79 data, version = None, None 123 data, version = None, None
80 124
81 if version == PARSERCACHE_VERSION: 125 if version != PARSERCACHE_VERSION:
82 for h in data[0]: 126 data = [{}, {}]
83 if h not in pythonparsecache: 127
84 pythonparsecache[h] = data[0][h] 128 for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]:
85 for h in data[1]: 129 f = os.path.join(os.path.dirname(cachefile), f)
86 if h not in pythonparsecache: 130 try:
87 shellparsecache[h] = data[1][h] 131 p = pickle.Unpickler(file(f, "rb"))
132 extradata, version = p.load()
133 except (IOError, EOFError):
134 extradata, version = [{}, {}], None
135
136 if version != PARSERCACHE_VERSION:
137 continue
138
139 for h in extradata[0]:
140 if h not in data[0]:
141 data[0][h] = extradata[0][h]
142 for h in extradata[1]:
143 if h not in data[1]:
144 data[1][h] = extradata[1][h]
145 os.unlink(f)
88 146
89 p = pickle.Pickler(file(cachefile, "wb"), -1) 147 p = pickle.Pickler(file(cachefile, "wb"), -1)
90 p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION]) 148 p.dump([data, PARSERCACHE_VERSION])
91 bb.utils.unlockfile(lf) 149
150 bb.utils.unlockfile(glf)
151
92 152
93class PythonParser(): 153class PythonParser():
94 class ValueVisitor(): 154 class ValueVisitor():