From 7c9444e9a58438058ec577bd8c2e51820c98072d Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Thu, 18 Nov 2010 22:47:36 -0700 Subject: cache: sync the cache file to disk in the background This version uses a thread rather than a process, to avoid problems with waitpid handling. This gives slightly less overall build time reduction than the separate process for it did (this reduces a -c compile coreutils-native by about 3 seconds, while the process reduced it by 7 seconds), however this time is quite insignificant relative to a typical build. The biggest issue with non-backgrounded syncing is the perceived delay before work begins, and this resolves that without breaking anything, or so it seems. (Bitbake rev: 5ab6c5c7b007b8c77c751582141afc07c183d672) Signed-off-by: Chris Larson Signed-off-by: Richard Purdie --- bitbake/lib/bb/cooker.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'bitbake/lib/bb/cooker.py') diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 0143c149b8..ac9758d402 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -25,6 +25,7 @@ from __future__ import print_function import sys, os, glob, os.path, re, time import logging import sre_constants +import threading import multiprocessing import signal from cStringIO import StringIO @@ -983,6 +984,7 @@ class CookerParser(object): # Internal data self.filelist = filelist self.cooker = cooker + self.cfgdata = cooker.configuration.data # Accounting statistics self.parsed = 0 @@ -1006,7 +1008,6 @@ class CookerParser(object): self.result_queue = multiprocessing.Queue() self.fromcache = [] - cfgdata = self.cooker.configuration.data for filename in self.filelist: appends = self.cooker.get_file_appends(filename) if not self.cooker.bb_cache.cacheValid(filename): @@ -1021,13 +1022,13 @@ class CookerParser(object): output.put(infos) self.processes = [] - num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or + num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or multiprocessing.cpu_count()) for i in xrange(num_processes): process = multiprocessing.Process(target=worker, args=(self.task_queue, self.result_queue, - cfgdata)) + self.cfgdata)) process.start() self.processes.append(process) @@ -1041,29 +1042,29 @@ class CookerParser(object): self.task_queue.close() for process in self.processes: process.join() - self.cooker.bb_cache.sync() - bb.codeparser.parser_cache_save(self.cooker.configuration.data) + threading.Thread(target=self.cooker.bb_cache.sync).start() + threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)).start() if self.error > 0: raise ParsingErrorsFound() - def progress(self): - bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, - self.skipped, self.masked, - self.virtuals, self.error, - self.total), - self.cooker.configuration.event_data) - def parse_next(self): cooker = self.cooker if self.current >= self.total: + event = bb.event.ParseCompleted(self.cached, self.parsed, + self.skipped, self.masked, + self.virtuals, self.error, + self.total) + bb.event.fire(event, self.cfgdata) self.shutdown() return False + elif self.current == 0: + bb.event.fire(bb.event.ParseStarted(self.total, self.skipped, self.masked), + self.cfgdata) try: if self.result_queue.empty() and self.fromcache: filename, appends = self.fromcache.pop() - _, infos = cooker.bb_cache.load(filename, appends, - self.cooker.configuration.data) + _, infos = cooker.bb_cache.load(filename, appends, self.cfgdata) parsed = False else: infos = self.result_queue.get() @@ -1087,7 +1088,7 @@ class CookerParser(object): if info.skipped: self.skipped += 1 finally: - self.progress() + bb.event.fire(bb.event.ParseProgress(self.current), self.cfgdata) self.current += 1 return True -- cgit v1.2.3-54-g00ecf