From 7c9444e9a58438058ec577bd8c2e51820c98072d Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Thu, 18 Nov 2010 22:47:36 -0700 Subject: cache: sync the cache file to disk in the background This version uses a thread rather than a process, to avoid problems with waitpid handling. This gives slightly less overall build time reduction than the separate process for it did (this reduces a -c compile coreutils-native by about 3 seconds, while the process reduced it by 7 seconds), however this time is quite insignificant relative to a typical build. The biggest issue with non-backgrounded syncing is the perceived delay before work begins, and this resolves that without breaking anything, or so it seems. (Bitbake rev: 5ab6c5c7b007b8c77c751582141afc07c183d672) Signed-off-by: Chris Larson Signed-off-by: Richard Purdie --- bitbake/lib/bb/cooker.py | 31 ++++++++++++++++--------------- bitbake/lib/bb/event.py | 20 ++++++++++++++++---- bitbake/lib/bb/ui/knotty.py | 34 +++++++++++++++++++--------------- bitbake/lib/bb/ui/uievent.py | 18 +++++++++--------- 4 files changed, 60 insertions(+), 43 deletions(-) diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 0143c149b8..ac9758d402 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -25,6 +25,7 @@ from __future__ import print_function import sys, os, glob, os.path, re, time import logging import sre_constants +import threading import multiprocessing import signal from cStringIO import StringIO @@ -983,6 +984,7 @@ class CookerParser(object): # Internal data self.filelist = filelist self.cooker = cooker + self.cfgdata = cooker.configuration.data # Accounting statistics self.parsed = 0 @@ -1006,7 +1008,6 @@ class CookerParser(object): self.result_queue = multiprocessing.Queue() self.fromcache = [] - cfgdata = self.cooker.configuration.data for filename in self.filelist: appends = self.cooker.get_file_appends(filename) if not self.cooker.bb_cache.cacheValid(filename): @@ -1021,13 +1022,13 @@ class CookerParser(object): output.put(infos) self.processes = [] - num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or + num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or multiprocessing.cpu_count()) for i in xrange(num_processes): process = multiprocessing.Process(target=worker, args=(self.task_queue, self.result_queue, - cfgdata)) + self.cfgdata)) process.start() self.processes.append(process) @@ -1041,29 +1042,29 @@ class CookerParser(object): self.task_queue.close() for process in self.processes: process.join() - self.cooker.bb_cache.sync() - bb.codeparser.parser_cache_save(self.cooker.configuration.data) + threading.Thread(target=self.cooker.bb_cache.sync).start() + threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)).start() if self.error > 0: raise ParsingErrorsFound() - def progress(self): - bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, - self.skipped, self.masked, - self.virtuals, self.error, - self.total), - self.cooker.configuration.event_data) - def parse_next(self): cooker = self.cooker if self.current >= self.total: + event = bb.event.ParseCompleted(self.cached, self.parsed, + self.skipped, self.masked, + self.virtuals, self.error, + self.total) + bb.event.fire(event, self.cfgdata) self.shutdown() return False + elif self.current == 0: + bb.event.fire(bb.event.ParseStarted(self.total, self.skipped, self.masked), + self.cfgdata) try: if self.result_queue.empty() and self.fromcache: filename, appends = self.fromcache.pop() - _, infos = cooker.bb_cache.load(filename, appends, - self.cooker.configuration.data) + _, infos = cooker.bb_cache.load(filename, appends, self.cfgdata) parsed = False else: infos = self.result_queue.get() @@ -1087,7 +1088,7 @@ class CookerParser(object): if info.skipped: self.skipped += 1 finally: - self.progress() + bb.event.fire(bb.event.ParseProgress(self.current), self.cfgdata) self.current += 1 return True diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py index 5f292bccbc..5b0a183acd 100644 --- a/bitbake/lib/bb/event.py +++ b/bitbake/lib/bb/event.py @@ -296,10 +296,16 @@ class MultipleProviders(Event): """ return self._candidates -class ParseProgress(Event): - """ - Parsing Progress Event - """ +class ParseStarted(Event): + """Recipe parsing for the runqueue has begun""" + def __init__(self, total, skipped, masked): + Event.__init__(self) + self.total = total + self.skipped = skipped + self.masked = masked + +class ParseCompleted(Event): + """Recipe parsing for the runqueue has completed""" def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total): Event.__init__(self) @@ -312,6 +318,12 @@ class ParseProgress(Event): self.sofar = cached + parsed self.total = total +class ParseProgress(Event): + """Recipe parsing progress""" + + def __init__(self, current): + self.current = current + class DepTreeGenerated(Event): """ Event when a dependency tree has been generated diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py index a34991bb68..326f164353 100644 --- a/bitbake/lib/bb/ui/knotty.py +++ b/bitbake/lib/bb/ui/knotty.py @@ -78,6 +78,7 @@ def init(server, eventHandler): return 1 pbar = None + interactive = os.isatty(sys.stdout.fileno()) shutdown = 0 return_value = 0 while True: @@ -132,23 +133,26 @@ def init(server, eventHandler): if isinstance(event, bb.build.TaskBase): logger.info(event._message) continue + if isinstance(event, bb.event.ParseStarted): + if interactive: + pbar = progressbar.ProgressBar(widgets=widgets, + maxval=event.total).start() + else: + sys.stdout.write("Parsing recipes...") + sys.stdout.flush() + continue if isinstance(event, bb.event.ParseProgress): - current, total = event.sofar, event.total - if os.isatty(sys.stdout.fileno()): - if not pbar: - pbar = progressbar.ProgressBar(widgets=widgets, - maxval=total).start() - pbar.update(current) + if interactive: + pbar.update(event.current) + continue + if isinstance(event, bb.event.ParseCompleted): + if interactive: + pbar.update(event.total) else: - if current == 1: - sys.stdout.write("Parsing .bb files, please wait...") - sys.stdout.flush() - if current == total: - sys.stdout.write("done.") - sys.stdout.flush() - if current == total: - print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." - % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))) + sys.stdout.write("done.\n") + sys.stdout.flush() + print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." + % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))) continue if isinstance(event, bb.command.CookerCommandCompleted): diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py index f1e4d791ee..b404805d81 100644 --- a/bitbake/lib/bb/ui/uievent.py +++ b/bitbake/lib/bb/ui/uievent.py @@ -37,8 +37,8 @@ class BBUIEventQueue: self.BBServer = BBServer self.t = threading.Thread() - self.t.setDaemon(True) - self.t.run = self.startCallbackHandler + self.t.setDaemon(True) + self.t.run = self.startCallbackHandler self.t.start() def getEvent(self): @@ -70,7 +70,7 @@ class BBUIEventQueue: def startCallbackHandler(self): server = UIXMLRPCServer() - self.host, self.port = server.socket.getsockname() + self.host, self.port = server.socket.getsockname() server.register_function( self.system_quit, "event.quit" ) server.register_function( self.queue_event, "event.send" ) @@ -83,7 +83,7 @@ class BBUIEventQueue: server.handle_request() server.server_close() - def system_quit( self ): + def system_quit( self ): """ Shut down the callback thread """ @@ -95,11 +95,11 @@ class BBUIEventQueue: class UIXMLRPCServer (SimpleXMLRPCServer): - def __init__( self, interface = ("localhost", 0) ): + def __init__( self, interface = ("localhost", 0) ): self.quit = False - SimpleXMLRPCServer.__init__( self, - interface, - requestHandler=SimpleXMLRPCRequestHandler, + SimpleXMLRPCServer.__init__( self, + interface, + requestHandler=SimpleXMLRPCRequestHandler, logRequests=False, allow_none=True) def get_request(self): @@ -121,4 +121,4 @@ class UIXMLRPCServer (SimpleXMLRPCServer): if request is None: return SimpleXMLRPCServer.process_request(self, request, client_address) - + -- cgit v1.2.3-54-g00ecf