bitbake/cooker/codeparser: Ensure the code parser cache is saved for each parsing process

Before this change, the codeparser cache was only being saved for the main
server process. This is suboptimal as it leaves code being re-evaluated at
task execution time and increases parse time.

We use the multiprocess Finalize() functionality to ensure each process
saves out its cache. We need to update the cache save function to be multiprocess
friendly with locking.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 15 insertions, 4 deletions

diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 84d1c09f21..fdba06f678 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -70,8 +70,22 @@ def parser_cache_save(d):
     if not cachefile:
         return
 
+    lf = bb.utils.lockfile(cachefile + ".lock")
+
+    p = pickle.Unpickler(file(cachefile, "rb"))
+    data, version = p.load()
+
+    if version == PARSERCACHE_VERSION:
+        for h in data[0]:
+            if h not in pythonparsecache:
+                pythonparsecache[h] = data[0][h]
+        for h in data[1]:
+            if h not in pythonparsecache:
+                shellparsecache[h] = data[1][h]
+
     p = pickle.Pickler(file(cachefile, "wb"), -1)
     p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
+    bb.utils.unlockfile(lf)
 
 class PythonParser():
     class ValueVisitor():
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 3c7b60eb95..98612656ab 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1112,6 +1112,7 @@ class CookerParser(object):
     def start(self):
         def init(cfg):
             signal.signal(signal.SIGINT, signal.SIG_IGN)
+            multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
             parse_file.cfg = cfg
 
         bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
@@ -1137,10 +1138,6 @@ class CookerParser(object):
         sync.start()
         atexit.register(lambda: sync.join())
 
-        codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data))
-        codesync.start()
-        atexit.register(lambda: codesync.join())
-
     def load_cached(self):
         for filename, appends in self.fromcache:
             cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)