diff options
author | Chris Larson <chris_larson@mentor.com> | 2010-11-18 20:21:54 -0700 |
---|---|---|
committer | Richard Purdie <rpurdie@linux.intel.com> | 2011-01-04 14:46:42 +0000 |
commit | 32ea7668712a50d8f8b67d5e4558039e5092a485 (patch) | |
tree | 2473f8b1aade6131c7a37fbad2cc4d23998a3a56 /bitbake/lib/bb/cooker.py | |
parent | 570bec37a898fb502d166a22f20bdb1da8c21c38 (diff) | |
download | poky-32ea7668712a50d8f8b67d5e4558039e5092a485.tar.gz |
Implement parallel parsing support
This utilizes python's multiprocessing module. The default number of threads
to be used is the same as the number of available processor cores, however,
you can manually set this with the BB_NUMBER_PARSE_THREADS variable.
(Bitbake rev: c7b3ec819549e51e438d293969e205883fee725f)
Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 135 |
1 files changed, 97 insertions, 38 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 6194919e4c..0143c149b8 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -25,6 +25,8 @@ from __future__ import print_function | |||
25 | import sys, os, glob, os.path, re, time | 25 | import sys, os, glob, os.path, re, time |
26 | import logging | 26 | import logging |
27 | import sre_constants | 27 | import sre_constants |
28 | import multiprocessing | ||
29 | import signal | ||
28 | from cStringIO import StringIO | 30 | from cStringIO import StringIO |
29 | from contextlib import closing | 31 | from contextlib import closing |
30 | import bb | 32 | import bb |
@@ -976,7 +978,7 @@ class CookerExit(bb.event.Event): | |||
976 | def __init__(self): | 978 | def __init__(self): |
977 | bb.event.Event.__init__(self) | 979 | bb.event.Event.__init__(self) |
978 | 980 | ||
979 | class CookerParser: | 981 | class CookerParser(object): |
980 | def __init__(self, cooker, filelist, masked): | 982 | def __init__(self, cooker, filelist, masked): |
981 | # Internal data | 983 | # Internal data |
982 | self.filelist = filelist | 984 | self.filelist = filelist |
@@ -987,49 +989,106 @@ class CookerParser: | |||
987 | self.cached = 0 | 989 | self.cached = 0 |
988 | self.error = 0 | 990 | self.error = 0 |
989 | self.masked = masked | 991 | self.masked = masked |
990 | self.total = len(filelist) | ||
991 | 992 | ||
992 | self.skipped = 0 | 993 | self.skipped = 0 |
993 | self.virtuals = 0 | 994 | self.virtuals = 0 |
995 | self.total = len(filelist) | ||
994 | 996 | ||
995 | # Pointer to the next file to parse | 997 | # current to the next file to parse |
996 | self.pointer = 0 | 998 | self.current = 0 |
997 | 999 | self.result_queue = None | |
998 | def parse_next(self): | 1000 | self.fromcache = None |
999 | cooker = self.cooker | ||
1000 | if self.pointer < len(self.filelist): | ||
1001 | f = self.filelist[self.pointer] | ||
1002 | |||
1003 | try: | ||
1004 | fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.get_file_appends(f), cooker.configuration.data, cooker.status) | ||
1005 | if fromCache: | ||
1006 | self.cached += 1 | ||
1007 | else: | ||
1008 | self.parsed += 1 | ||
1009 | |||
1010 | self.skipped += skipped | ||
1011 | self.virtuals += virtuals | ||
1012 | 1001 | ||
1013 | except KeyboardInterrupt: | 1002 | self.launch_processes() |
1014 | cooker.bb_cache.remove(f) | ||
1015 | cooker.bb_cache.sync() | ||
1016 | raise | ||
1017 | except Exception as e: | ||
1018 | self.error += 1 | ||
1019 | cooker.bb_cache.remove(f) | ||
1020 | parselog.exception("Unable to open %s", f) | ||
1021 | except: | ||
1022 | cooker.bb_cache.remove(f) | ||
1023 | raise | ||
1024 | finally: | ||
1025 | bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data) | ||
1026 | 1003 | ||
1027 | self.pointer += 1 | 1004 | def launch_processes(self): |
1005 | self.task_queue = multiprocessing.Queue() | ||
1006 | self.result_queue = multiprocessing.Queue() | ||
1007 | |||
1008 | self.fromcache = [] | ||
1009 | cfgdata = self.cooker.configuration.data | ||
1010 | for filename in self.filelist: | ||
1011 | appends = self.cooker.get_file_appends(filename) | ||
1012 | if not self.cooker.bb_cache.cacheValid(filename): | ||
1013 | self.task_queue.put((filename, appends)) | ||
1014 | else: | ||
1015 | self.fromcache.append((filename, appends)) | ||
1016 | |||
1017 | def worker(input, output, cfgdata): | ||
1018 | signal.signal(signal.SIGINT, signal.SIG_IGN) | ||
1019 | for filename, appends in iter(input.get, 'STOP'): | ||
1020 | infos = bb.cache.Cache.parse(filename, appends, cfgdata) | ||
1021 | output.put(infos) | ||
1022 | |||
1023 | self.processes = [] | ||
1024 | num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or | ||
1025 | multiprocessing.cpu_count()) | ||
1026 | for i in xrange(num_processes): | ||
1027 | process = multiprocessing.Process(target=worker, | ||
1028 | args=(self.task_queue, | ||
1029 | self.result_queue, | ||
1030 | cfgdata)) | ||
1031 | process.start() | ||
1032 | self.processes.append(process) | ||
1033 | |||
1034 | def shutdown(self, clean=True): | ||
1035 | self.result_queue.close() | ||
1036 | for process in self.processes: | ||
1037 | if clean: | ||
1038 | self.task_queue.put('STOP') | ||
1039 | else: | ||
1040 | process.terminate() | ||
1041 | self.task_queue.close() | ||
1042 | for process in self.processes: | ||
1043 | process.join() | ||
1044 | self.cooker.bb_cache.sync() | ||
1045 | bb.codeparser.parser_cache_save(self.cooker.configuration.data) | ||
1046 | if self.error > 0: | ||
1047 | raise ParsingErrorsFound() | ||
1048 | |||
1049 | def progress(self): | ||
1050 | bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, | ||
1051 | self.skipped, self.masked, | ||
1052 | self.virtuals, self.error, | ||
1053 | self.total), | ||
1054 | self.cooker.configuration.event_data) | ||
1028 | 1055 | ||
1029 | if self.pointer >= self.total: | 1056 | def parse_next(self): |
1030 | cooker.bb_cache.sync() | 1057 | cooker = self.cooker |
1031 | bb.codeparser.parser_cache_save(cooker.configuration.data) | 1058 | if self.current >= self.total: |
1032 | if self.error > 0: | 1059 | self.shutdown() |
1033 | raise ParsingErrorsFound | ||
1034 | return False | 1060 | return False |
1061 | |||
1062 | try: | ||
1063 | if self.result_queue.empty() and self.fromcache: | ||
1064 | filename, appends = self.fromcache.pop() | ||
1065 | _, infos = cooker.bb_cache.load(filename, appends, | ||
1066 | self.cooker.configuration.data) | ||
1067 | parsed = False | ||
1068 | else: | ||
1069 | infos = self.result_queue.get() | ||
1070 | parsed = True | ||
1071 | except KeyboardInterrupt: | ||
1072 | self.shutdown(clean=False) | ||
1073 | raise | ||
1074 | except Exception as e: | ||
1075 | self.error += 1 | ||
1076 | parselog.critical(str(e)) | ||
1077 | else: | ||
1078 | if parsed: | ||
1079 | self.parsed += 1 | ||
1080 | else: | ||
1081 | self.cached += 1 | ||
1082 | self.virtuals += len(infos) | ||
1083 | |||
1084 | for virtualfn, info in infos: | ||
1085 | cooker.bb_cache.add_info(virtualfn, info, cooker.status, | ||
1086 | parsed=parsed) | ||
1087 | if info.skipped: | ||
1088 | self.skipped += 1 | ||
1089 | finally: | ||
1090 | self.progress() | ||
1091 | |||
1092 | self.current += 1 | ||
1035 | return True | 1093 | return True |
1094 | |||