summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2011-04-10 10:52:29 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-05-05 15:52:53 +0100
commitc8928e93dd8a08d6768623f6491c9ba31d0aa06f (patch)
tree3ece2e5b224101661bfc889226bb94be06cdbc82
parent9211fd9c375489c73924fd43f1f8a0da2c4290bb (diff)
downloadpoky-c8928e93dd8a08d6768623f6491c9ba31d0aa06f.tar.gz
bitbake/cooker/codeparser: Ensure the code parser cache is saved for each parsing process
Before this change, the codeparser cache was only being saved for the main server process. This is suboptimal as it leaves code being re-evaluated at task execution time and increases parse time. We use the multiprocess Finalize() functionality to ensure each process saves out its cache. We need to update the cache save function to be multiprocess friendly with locking. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/codeparser.py14
-rw-r--r--bitbake/lib/bb/cooker.py5
2 files changed, 15 insertions, 4 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 84d1c09f21..fdba06f678 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -70,8 +70,22 @@ def parser_cache_save(d):
70 if not cachefile: 70 if not cachefile:
71 return 71 return
72 72
73 lf = bb.utils.lockfile(cachefile + ".lock")
74
75 p = pickle.Unpickler(file(cachefile, "rb"))
76 data, version = p.load()
77
78 if version == PARSERCACHE_VERSION:
79 for h in data[0]:
80 if h not in pythonparsecache:
81 pythonparsecache[h] = data[0][h]
82 for h in data[1]:
83 if h not in pythonparsecache:
84 shellparsecache[h] = data[1][h]
85
73 p = pickle.Pickler(file(cachefile, "wb"), -1) 86 p = pickle.Pickler(file(cachefile, "wb"), -1)
74 p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION]) 87 p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
88 bb.utils.unlockfile(lf)
75 89
76class PythonParser(): 90class PythonParser():
77 class ValueVisitor(): 91 class ValueVisitor():
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 3c7b60eb95..98612656ab 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1112,6 +1112,7 @@ class CookerParser(object):
1112 def start(self): 1112 def start(self):
1113 def init(cfg): 1113 def init(cfg):
1114 signal.signal(signal.SIGINT, signal.SIG_IGN) 1114 signal.signal(signal.SIGINT, signal.SIG_IGN)
1115 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
1115 parse_file.cfg = cfg 1116 parse_file.cfg = cfg
1116 1117
1117 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) 1118 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
@@ -1137,10 +1138,6 @@ class CookerParser(object):
1137 sync.start() 1138 sync.start()
1138 atexit.register(lambda: sync.join()) 1139 atexit.register(lambda: sync.join())
1139 1140
1140 codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data))
1141 codesync.start()
1142 atexit.register(lambda: codesync.join())
1143
1144 def load_cached(self): 1141 def load_cached(self):
1145 for filename, appends in self.fromcache: 1142 for filename, appends in self.fromcache:
1146 cached, infos = self.bb_cache.load(filename, appends, self.cfgdata) 1143 cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)