summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorChris Larson <chris_larson@mentor.com>2010-12-07 13:00:22 -0500
committerRichard Purdie <rpurdie@linux.intel.com>2011-01-04 14:46:46 +0000
commitac4d926f413b127810130ad7a442e6daf1fdd1b9 (patch)
tree93da0b9b625a11d06a710ad71124c281d672b69f /bitbake
parente5624a4ed30c33440fd2e421aa93bda0476fae5a (diff)
downloadpoky-ac4d926f413b127810130ad7a442e6daf1fdd1b9.tar.gz
cooker: use a pool, abort on first parse error
(Bitbake rev: 9caf65e79f95fe0045e727391e974c4c1e7411ff) Signed-off-by: Chris Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/cooker.py144
1 files changed, 60 insertions, 84 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 18a22de711..bf896e9428 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -23,12 +23,13 @@
23 23
24from __future__ import print_function 24from __future__ import print_function
25import sys, os, glob, os.path, re, time 25import sys, os, glob, os.path, re, time
26import atexit
27import itertools
26import logging 28import logging
27import sre_constants
28import threading
29import multiprocessing 29import multiprocessing
30import signal 30import signal
31import atexit 31import sre_constants
32import threading
32from cStringIO import StringIO 33from cStringIO import StringIO
33from contextlib import closing 34from contextlib import closing
34import bb 35import bb
@@ -45,11 +46,6 @@ class MultipleMatches(Exception):
45 Exception raised when multiple file matches are found 46 Exception raised when multiple file matches are found
46 """ 47 """
47 48
48class ParsingErrorsFound(Exception):
49 """
50 Exception raised when parsing errors are found
51 """
52
53class NothingToBuild(Exception): 49class NothingToBuild(Exception):
54 """ 50 """
55 Exception raised when there is nothing to build 51 Exception raised when there is nothing to build
@@ -976,6 +972,10 @@ class CookerExit(bb.event.Event):
976 def __init__(self): 972 def __init__(self):
977 bb.event.Event.__init__(self) 973 bb.event.Event.__init__(self)
978 974
975def parse_file(task):
976 filename, appends = task
977 return True, bb.cache.Cache.parse(filename, appends, parse_file.cfg)
978
979class CookerParser(object): 979class CookerParser(object):
980 def __init__(self, cooker, filelist, masked): 980 def __init__(self, cooker, filelist, masked):
981 self.filelist = filelist 981 self.filelist = filelist
@@ -993,113 +993,89 @@ class CookerParser(object):
993 self.total = len(filelist) 993 self.total = len(filelist)
994 994
995 self.current = 0 995 self.current = 0
996 self.bb_cache = None
997 self.task_queue = None
998 self.result_queue = None
999 self.fromcache = None
1000 self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or 996 self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
1001 multiprocessing.cpu_count()) 997 multiprocessing.cpu_count())
1002 998
1003 def launch_processes(self): 999 self.bb_cache = bb.cache.Cache(self.cfgdata)
1004 self.task_queue = multiprocessing.Queue()
1005 self.result_queue = multiprocessing.Queue()
1006
1007 self.fromcache = [] 1000 self.fromcache = []
1001 self.willparse = []
1008 for filename in self.filelist: 1002 for filename in self.filelist:
1009 appends = self.cooker.get_file_appends(filename) 1003 appends = self.cooker.get_file_appends(filename)
1010 if not self.bb_cache.cacheValid(filename): 1004 if not self.bb_cache.cacheValid(filename):
1011 self.task_queue.put((filename, appends)) 1005 self.willparse.append((filename, appends))
1012 else: 1006 else:
1013 self.fromcache.append((filename, appends)) 1007 self.fromcache.append((filename, appends))
1014 self.toparse = self.total - len(self.fromcache) 1008 self.toparse = self.total - len(self.fromcache)
1015 self.progress_chunk = max(self.toparse / 100, 1) 1009 self.progress_chunk = max(self.toparse / 100, 1)
1016 1010
1017 def worker(input, output, cfgdata): 1011 self.start()
1012
1013 def start(self):
1014 def init(cfg):
1018 signal.signal(signal.SIGINT, signal.SIG_IGN) 1015 signal.signal(signal.SIGINT, signal.SIG_IGN)
1019 for filename, appends in iter(input.get, 'STOP'): 1016 parse_file.cfg = cfg
1020 try:
1021 infos = bb.cache.Cache.parse(filename, appends, cfgdata)
1022 except bb.parse.ParseError as exc:
1023 output.put(exc)
1024 else:
1025 output.put(infos)
1026 1017
1027 self.processes = [] 1018 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
1028 for i in xrange(self.num_processes): 1019
1029 process = multiprocessing.Process(target=worker, 1020 self.pool = multiprocessing.Pool(self.num_processes, init, [self.cfgdata])
1030 args=(self.task_queue, 1021 parsed = self.pool.imap(parse_file, self.willparse)
1031 self.result_queue, 1022 self.pool.close()
1032 self.cfgdata)) 1023
1033 process.start() 1024 self.results = itertools.chain(self.load_cached(), parsed)
1034 self.processes.append(process)
1035 1025
1036 def shutdown(self, clean=True): 1026 def shutdown(self, clean=True):
1037 self.result_queue.close() 1027 if clean:
1038 for process in self.processes: 1028 event = bb.event.ParseCompleted(self.cached, self.parsed,
1039 if clean: 1029 self.skipped, self.masked,
1040 self.task_queue.put('STOP') 1030 self.virtuals, self.error,
1041 else: 1031 self.total)
1042 process.terminate() 1032 bb.event.fire(event, self.cfgdata)
1043 self.task_queue.close() 1033 else:
1044 for process in self.processes: 1034 self.pool.terminate()
1045 process.join() 1035 self.pool.join()
1036
1046 sync = threading.Thread(target=self.bb_cache.sync) 1037 sync = threading.Thread(target=self.bb_cache.sync)
1047 sync.start() 1038 sync.start()
1048 atexit.register(lambda: sync.join()) 1039 atexit.register(lambda: sync.join())
1040
1049 codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)) 1041 codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data))
1050 codesync.start() 1042 codesync.start()
1051 atexit.register(lambda: codesync.join()) 1043 atexit.register(lambda: codesync.join())
1052 if self.error > 0: 1044
1053 raise ParsingErrorsFound() 1045 def load_cached(self):
1046 for filename, appends in self.fromcache:
1047 cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
1048 yield not cached, infos
1054 1049
1055 def parse_next(self): 1050 def parse_next(self):
1056 if self.current >= self.total: 1051 try:
1057 event = bb.event.ParseCompleted(self.cached, self.parsed, 1052 parsed, result = self.results.next()
1058 self.skipped, self.masked, 1053 except StopIteration:
1059 self.virtuals, self.error,
1060 self.total)
1061 bb.event.fire(event, self.cfgdata)
1062 self.shutdown() 1054 self.shutdown()
1063 return False 1055 return False
1064 elif not self.bb_cache:
1065 self.bb_cache = bb.cache.Cache(self.cfgdata)
1066 self.launch_processes()
1067 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
1068 return True
1069
1070 try:
1071 if self.result_queue.empty() and self.fromcache:
1072 filename, appends = self.fromcache.pop()
1073 _, result = self.bb_cache.load(filename, appends, self.cfgdata)
1074 parsed = False
1075 self.cached += 1
1076 else:
1077 result = self.result_queue.get()
1078 if isinstance(result, Exception):
1079 raise result
1080
1081 parsed = True
1082 self.parsed += 1
1083 if self.parsed % self.progress_chunk == 0:
1084 bb.event.fire(bb.event.ParseProgress(self.parsed),
1085 self.cfgdata)
1086 except KeyboardInterrupt: 1056 except KeyboardInterrupt:
1087 self.shutdown(clean=False) 1057 self.shutdown(clean=False)
1088 raise 1058 raise
1089 except Exception as e: 1059 except Exception as exc:
1090 self.error += 1 1060 self.shutdown(clean=False)
1091 parselog.critical(str(e)) 1061 sys.exit(1)
1092 else:
1093 self.virtuals += len(result)
1094
1095 for virtualfn, info in result:
1096 if info.skipped:
1097 self.skipped += 1
1098 else:
1099 self.bb_cache.add_info(virtualfn, info, self.cooker.status,
1100 parsed=parsed)
1101 1062
1102 self.current += 1 1063 self.current += 1
1064 self.virtuals += len(result)
1065 if parsed:
1066 self.parsed += 1
1067 if self.parsed % self.progress_chunk == 0:
1068 bb.event.fire(bb.event.ParseProgress(self.parsed),
1069 self.cfgdata)
1070 else:
1071 self.cached += 1
1072
1073 for virtualfn, info in result:
1074 if info.skipped:
1075 self.skipped += 1
1076 else:
1077 self.bb_cache.add_info(virtualfn, info, self.cooker.status,
1078 parsed=parsed)
1103 return True 1079 return True
1104 1080
1105 def reparse(self, filename): 1081 def reparse(self, filename):