diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 135 |
1 files changed, 97 insertions, 38 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 6194919e4c..0143c149b8 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -25,6 +25,8 @@ from __future__ import print_function | |||
25 | import sys, os, glob, os.path, re, time | 25 | import sys, os, glob, os.path, re, time |
26 | import logging | 26 | import logging |
27 | import sre_constants | 27 | import sre_constants |
28 | import multiprocessing | ||
29 | import signal | ||
28 | from cStringIO import StringIO | 30 | from cStringIO import StringIO |
29 | from contextlib import closing | 31 | from contextlib import closing |
30 | import bb | 32 | import bb |
@@ -976,7 +978,7 @@ class CookerExit(bb.event.Event): | |||
976 | def __init__(self): | 978 | def __init__(self): |
977 | bb.event.Event.__init__(self) | 979 | bb.event.Event.__init__(self) |
978 | 980 | ||
979 | class CookerParser: | 981 | class CookerParser(object): |
980 | def __init__(self, cooker, filelist, masked): | 982 | def __init__(self, cooker, filelist, masked): |
981 | # Internal data | 983 | # Internal data |
982 | self.filelist = filelist | 984 | self.filelist = filelist |
@@ -987,49 +989,106 @@ class CookerParser: | |||
987 | self.cached = 0 | 989 | self.cached = 0 |
988 | self.error = 0 | 990 | self.error = 0 |
989 | self.masked = masked | 991 | self.masked = masked |
990 | self.total = len(filelist) | ||
991 | 992 | ||
992 | self.skipped = 0 | 993 | self.skipped = 0 |
993 | self.virtuals = 0 | 994 | self.virtuals = 0 |
995 | self.total = len(filelist) | ||
994 | 996 | ||
995 | # Pointer to the next file to parse | 997 | # current to the next file to parse |
996 | self.pointer = 0 | 998 | self.current = 0 |
997 | 999 | self.result_queue = None | |
998 | def parse_next(self): | 1000 | self.fromcache = None |
999 | cooker = self.cooker | ||
1000 | if self.pointer < len(self.filelist): | ||
1001 | f = self.filelist[self.pointer] | ||
1002 | |||
1003 | try: | ||
1004 | fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.get_file_appends(f), cooker.configuration.data, cooker.status) | ||
1005 | if fromCache: | ||
1006 | self.cached += 1 | ||
1007 | else: | ||
1008 | self.parsed += 1 | ||
1009 | |||
1010 | self.skipped += skipped | ||
1011 | self.virtuals += virtuals | ||
1012 | 1001 | ||
1013 | except KeyboardInterrupt: | 1002 | self.launch_processes() |
1014 | cooker.bb_cache.remove(f) | ||
1015 | cooker.bb_cache.sync() | ||
1016 | raise | ||
1017 | except Exception as e: | ||
1018 | self.error += 1 | ||
1019 | cooker.bb_cache.remove(f) | ||
1020 | parselog.exception("Unable to open %s", f) | ||
1021 | except: | ||
1022 | cooker.bb_cache.remove(f) | ||
1023 | raise | ||
1024 | finally: | ||
1025 | bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data) | ||
1026 | 1003 | ||
1027 | self.pointer += 1 | 1004 | def launch_processes(self): |
1005 | self.task_queue = multiprocessing.Queue() | ||
1006 | self.result_queue = multiprocessing.Queue() | ||
1007 | |||
1008 | self.fromcache = [] | ||
1009 | cfgdata = self.cooker.configuration.data | ||
1010 | for filename in self.filelist: | ||
1011 | appends = self.cooker.get_file_appends(filename) | ||
1012 | if not self.cooker.bb_cache.cacheValid(filename): | ||
1013 | self.task_queue.put((filename, appends)) | ||
1014 | else: | ||
1015 | self.fromcache.append((filename, appends)) | ||
1016 | |||
1017 | def worker(input, output, cfgdata): | ||
1018 | signal.signal(signal.SIGINT, signal.SIG_IGN) | ||
1019 | for filename, appends in iter(input.get, 'STOP'): | ||
1020 | infos = bb.cache.Cache.parse(filename, appends, cfgdata) | ||
1021 | output.put(infos) | ||
1022 | |||
1023 | self.processes = [] | ||
1024 | num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or | ||
1025 | multiprocessing.cpu_count()) | ||
1026 | for i in xrange(num_processes): | ||
1027 | process = multiprocessing.Process(target=worker, | ||
1028 | args=(self.task_queue, | ||
1029 | self.result_queue, | ||
1030 | cfgdata)) | ||
1031 | process.start() | ||
1032 | self.processes.append(process) | ||
1033 | |||
1034 | def shutdown(self, clean=True): | ||
1035 | self.result_queue.close() | ||
1036 | for process in self.processes: | ||
1037 | if clean: | ||
1038 | self.task_queue.put('STOP') | ||
1039 | else: | ||
1040 | process.terminate() | ||
1041 | self.task_queue.close() | ||
1042 | for process in self.processes: | ||
1043 | process.join() | ||
1044 | self.cooker.bb_cache.sync() | ||
1045 | bb.codeparser.parser_cache_save(self.cooker.configuration.data) | ||
1046 | if self.error > 0: | ||
1047 | raise ParsingErrorsFound() | ||
1048 | |||
1049 | def progress(self): | ||
1050 | bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, | ||
1051 | self.skipped, self.masked, | ||
1052 | self.virtuals, self.error, | ||
1053 | self.total), | ||
1054 | self.cooker.configuration.event_data) | ||
1028 | 1055 | ||
1029 | if self.pointer >= self.total: | 1056 | def parse_next(self): |
1030 | cooker.bb_cache.sync() | 1057 | cooker = self.cooker |
1031 | bb.codeparser.parser_cache_save(cooker.configuration.data) | 1058 | if self.current >= self.total: |
1032 | if self.error > 0: | 1059 | self.shutdown() |
1033 | raise ParsingErrorsFound | ||
1034 | return False | 1060 | return False |
1061 | |||
1062 | try: | ||
1063 | if self.result_queue.empty() and self.fromcache: | ||
1064 | filename, appends = self.fromcache.pop() | ||
1065 | _, infos = cooker.bb_cache.load(filename, appends, | ||
1066 | self.cooker.configuration.data) | ||
1067 | parsed = False | ||
1068 | else: | ||
1069 | infos = self.result_queue.get() | ||
1070 | parsed = True | ||
1071 | except KeyboardInterrupt: | ||
1072 | self.shutdown(clean=False) | ||
1073 | raise | ||
1074 | except Exception as e: | ||
1075 | self.error += 1 | ||
1076 | parselog.critical(str(e)) | ||
1077 | else: | ||
1078 | if parsed: | ||
1079 | self.parsed += 1 | ||
1080 | else: | ||
1081 | self.cached += 1 | ||
1082 | self.virtuals += len(infos) | ||
1083 | |||
1084 | for virtualfn, info in infos: | ||
1085 | cooker.bb_cache.add_info(virtualfn, info, cooker.status, | ||
1086 | parsed=parsed) | ||
1087 | if info.skipped: | ||
1088 | self.skipped += 1 | ||
1089 | finally: | ||
1090 | self.progress() | ||
1091 | |||
1092 | self.current += 1 | ||
1035 | return True | 1093 | return True |
1094 | |||