diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 139 |
1 files changed, 82 insertions, 57 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index e527e23114..8f45233c8d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -525,7 +525,7 @@ class BBCooker: | |||
525 | self.parseConfiguration() | 525 | self.parseConfiguration() |
526 | 526 | ||
527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
528 | fn = self.matchFile(fn) | 528 | fn = self.matchFile(fn, mc) |
529 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 529 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
530 | elif len(pkgs_to_build) == 1: | 530 | elif len(pkgs_to_build) == 1: |
531 | mc = mc_base(pkgs_to_build[0]) | 531 | mc = mc_base(pkgs_to_build[0]) |
@@ -542,7 +542,7 @@ class BBCooker: | |||
542 | if fn: | 542 | if fn: |
543 | try: | 543 | try: |
544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
545 | envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) | 545 | envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn)) |
546 | except Exception as e: | 546 | except Exception as e: |
547 | parselog.exception("Unable to read %s", fn) | 547 | parselog.exception("Unable to read %s", fn) |
548 | raise | 548 | raise |
@@ -929,26 +929,33 @@ class BBCooker: | |||
929 | logger.info("Task dependencies saved to 'task-depends.dot'") | 929 | logger.info("Task dependencies saved to 'task-depends.dot'") |
930 | 930 | ||
931 | def show_appends_with_no_recipes(self): | 931 | def show_appends_with_no_recipes(self): |
932 | appends_without_recipes = {} | ||
932 | # Determine which bbappends haven't been applied | 933 | # Determine which bbappends haven't been applied |
933 | 934 | for mc in self.multiconfigs: | |
934 | # First get list of recipes, including skipped | 935 | # First get list of recipes, including skipped |
935 | recipefns = list(self.recipecaches[''].pkg_fn.keys()) | 936 | recipefns = list(self.recipecaches[mc].pkg_fn.keys()) |
936 | recipefns.extend(self.skiplist.keys()) | 937 | recipefns.extend(self.skiplist.keys()) |
937 | 938 | ||
938 | # Work out list of bbappends that have been applied | 939 | # Work out list of bbappends that have been applied |
939 | applied_appends = [] | 940 | applied_appends = [] |
940 | for fn in recipefns: | 941 | for fn in recipefns: |
941 | applied_appends.extend(self.collection.get_file_appends(fn)) | 942 | applied_appends.extend(self.collections[mc].get_file_appends(fn)) |
942 | 943 | ||
943 | appends_without_recipes = [] | 944 | appends_without_recipes[mc] = [] |
944 | for _, appendfn in self.collection.bbappends: | 945 | for _, appendfn in self.collections[mc].bbappends: |
945 | if not appendfn in applied_appends: | 946 | if not appendfn in applied_appends: |
946 | appends_without_recipes.append(appendfn) | 947 | appends_without_recipes[mc].append(appendfn) |
947 | 948 | ||
948 | if appends_without_recipes: | 949 | msgs = [] |
949 | msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) | 950 | for mc in sorted(appends_without_recipes.keys()): |
950 | warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | 951 | if appends_without_recipes[mc]: |
951 | False) or "no" | 952 | msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', |
953 | '\n '.join(appends_without_recipes[mc]))) | ||
954 | |||
955 | if msgs: | ||
956 | msg = "\n".join(msgs) | ||
957 | warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | ||
958 | False) or "no" | ||
952 | if warn_only.lower() in ("1", "yes", "true"): | 959 | if warn_only.lower() in ("1", "yes", "true"): |
953 | bb.warn(msg) | 960 | bb.warn(msg) |
954 | else: | 961 | else: |
@@ -1249,15 +1256,15 @@ class BBCooker: | |||
1249 | if siggen_cache: | 1256 | if siggen_cache: |
1250 | bb.parse.siggen.checksum_cache.mtime_cache.clear() | 1257 | bb.parse.siggen.checksum_cache.mtime_cache.clear() |
1251 | 1258 | ||
1252 | def matchFiles(self, bf): | 1259 | def matchFiles(self, bf, mc=''): |
1253 | """ | 1260 | """ |
1254 | Find the .bb files which match the expression in 'buildfile'. | 1261 | Find the .bb files which match the expression in 'buildfile'. |
1255 | """ | 1262 | """ |
1256 | if bf.startswith("/") or bf.startswith("../"): | 1263 | if bf.startswith("/") or bf.startswith("../"): |
1257 | bf = os.path.abspath(bf) | 1264 | bf = os.path.abspath(bf) |
1258 | 1265 | ||
1259 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1266 | self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} |
1260 | filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) | 1267 | filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) |
1261 | try: | 1268 | try: |
1262 | os.stat(bf) | 1269 | os.stat(bf) |
1263 | bf = os.path.abspath(bf) | 1270 | bf = os.path.abspath(bf) |
@@ -1270,12 +1277,12 @@ class BBCooker: | |||
1270 | matches.append(f) | 1277 | matches.append(f) |
1271 | return matches | 1278 | return matches |
1272 | 1279 | ||
1273 | def matchFile(self, buildfile): | 1280 | def matchFile(self, buildfile, mc=''): |
1274 | """ | 1281 | """ |
1275 | Find the .bb file which matches the expression in 'buildfile'. | 1282 | Find the .bb file which matches the expression in 'buildfile'. |
1276 | Raise an error if multiple files | 1283 | Raise an error if multiple files |
1277 | """ | 1284 | """ |
1278 | matches = self.matchFiles(buildfile) | 1285 | matches = self.matchFiles(buildfile, mc) |
1279 | if len(matches) != 1: | 1286 | if len(matches) != 1: |
1280 | if matches: | 1287 | if matches: |
1281 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) | 1288 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) |
@@ -1316,14 +1323,14 @@ class BBCooker: | |||
1316 | task = "do_%s" % task | 1323 | task = "do_%s" % task |
1317 | 1324 | ||
1318 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 1325 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
1319 | fn = self.matchFile(fn) | 1326 | fn = self.matchFile(fn, mc) |
1320 | 1327 | ||
1321 | self.buildSetVars() | 1328 | self.buildSetVars() |
1322 | self.reset_mtime_caches() | 1329 | self.reset_mtime_caches() |
1323 | 1330 | ||
1324 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 1331 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
1325 | 1332 | ||
1326 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) | 1333 | infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn)) |
1327 | infos = dict(infos) | 1334 | infos = dict(infos) |
1328 | 1335 | ||
1329 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 1336 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
@@ -1552,14 +1559,24 @@ class BBCooker: | |||
1552 | for dep in self.configuration.extra_assume_provided: | 1559 | for dep in self.configuration.extra_assume_provided: |
1553 | self.recipecaches[mc].ignored_dependencies.add(dep) | 1560 | self.recipecaches[mc].ignored_dependencies.add(dep) |
1554 | 1561 | ||
1555 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1562 | self.collections = {} |
1556 | (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) | 1563 | |
1564 | mcfilelist = {} | ||
1565 | total_masked = 0 | ||
1566 | searchdirs = set() | ||
1567 | for mc in self.multiconfigs: | ||
1568 | self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) | ||
1569 | (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) | ||
1570 | |||
1571 | mcfilelist[mc] = filelist | ||
1572 | total_masked += masked | ||
1573 | searchdirs |= set(search) | ||
1557 | 1574 | ||
1558 | # Add inotify watches for directories searched for bb/bbappend files | 1575 | # Add inotify watches for directories searched for bb/bbappend files |
1559 | for dirent in searchdirs: | 1576 | for dirent in searchdirs: |
1560 | self.add_filewatch([[dirent]], dirs=True) | 1577 | self.add_filewatch([[dirent]], dirs=True) |
1561 | 1578 | ||
1562 | self.parser = CookerParser(self, filelist, masked) | 1579 | self.parser = CookerParser(self, mcfilelist, total_masked) |
1563 | self.parsecache_valid = True | 1580 | self.parsecache_valid = True |
1564 | 1581 | ||
1565 | self.state = state.parsing | 1582 | self.state = state.parsing |
@@ -1571,7 +1588,7 @@ class BBCooker: | |||
1571 | self.show_appends_with_no_recipes() | 1588 | self.show_appends_with_no_recipes() |
1572 | self.handlePrefProviders() | 1589 | self.handlePrefProviders() |
1573 | for mc in self.multiconfigs: | 1590 | for mc in self.multiconfigs: |
1574 | self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) | 1591 | self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) |
1575 | self.state = state.running | 1592 | self.state = state.running |
1576 | 1593 | ||
1577 | # Send an event listing all stamps reachable after parsing | 1594 | # Send an event listing all stamps reachable after parsing |
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event): | |||
1679 | 1696 | ||
1680 | 1697 | ||
1681 | class CookerCollectFiles(object): | 1698 | class CookerCollectFiles(object): |
1682 | def __init__(self, priorities): | 1699 | def __init__(self, priorities, mc=''): |
1700 | self.mc = mc | ||
1683 | self.bbappends = [] | 1701 | self.bbappends = [] |
1684 | # Priorities is a list of tupples, with the second element as the pattern. | 1702 | # Priorities is a list of tupples, with the second element as the pattern. |
1685 | # We need to sort the list with the longest pattern first, and so on to | 1703 | # We need to sort the list with the longest pattern first, and so on to |
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object): | |||
1846 | (bbappend, filename) = b | 1864 | (bbappend, filename) = b |
1847 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): | 1865 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): |
1848 | filelist.append(filename) | 1866 | filelist.append(filename) |
1849 | return filelist | 1867 | return tuple(filelist) |
1850 | 1868 | ||
1851 | def collection_priorities(self, pkgfns, d): | 1869 | def collection_priorities(self, pkgfns, d): |
1852 | 1870 | ||
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object): | |||
1882 | for collection, pattern, regex, _ in self.bbfile_config_priorities: | 1900 | for collection, pattern, regex, _ in self.bbfile_config_priorities: |
1883 | if regex in unmatched: | 1901 | if regex in unmatched: |
1884 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': | 1902 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': |
1885 | collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) | 1903 | collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', |
1904 | collection, pattern)) | ||
1886 | 1905 | ||
1887 | return priorities | 1906 | return priorities |
1888 | 1907 | ||
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process): | |||
1978 | bb.event.LogHandler.filter = origfilter | 1997 | bb.event.LogHandler.filter = origfilter |
1979 | 1998 | ||
1980 | class CookerParser(object): | 1999 | class CookerParser(object): |
1981 | def __init__(self, cooker, filelist, masked): | 2000 | def __init__(self, cooker, mcfilelist, masked): |
1982 | self.filelist = filelist | 2001 | self.mcfilelist = mcfilelist |
1983 | self.cooker = cooker | 2002 | self.cooker = cooker |
1984 | self.cfgdata = cooker.data | 2003 | self.cfgdata = cooker.data |
1985 | self.cfghash = cooker.data_hash | 2004 | self.cfghash = cooker.data_hash |
@@ -1993,25 +2012,27 @@ class CookerParser(object): | |||
1993 | 2012 | ||
1994 | self.skipped = 0 | 2013 | self.skipped = 0 |
1995 | self.virtuals = 0 | 2014 | self.virtuals = 0 |
1996 | self.total = len(filelist) | ||
1997 | 2015 | ||
1998 | self.current = 0 | 2016 | self.current = 0 |
1999 | self.process_names = [] | 2017 | self.process_names = [] |
2000 | 2018 | ||
2001 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) | 2019 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) |
2002 | self.fromcache = [] | 2020 | self.fromcache = set() |
2003 | self.willparse = [] | 2021 | self.willparse = set() |
2004 | for filename in self.filelist: | 2022 | for mc in self.cooker.multiconfigs: |
2005 | appends = self.cooker.collection.get_file_appends(filename) | 2023 | for filename in self.mcfilelist[mc]: |
2006 | if not self.bb_cache.cacheValid(filename, appends): | 2024 | appends = self.cooker.collections[mc].get_file_appends(filename) |
2007 | self.willparse.append((filename, appends)) | 2025 | if not self.bb_cache.cacheValid(filename, appends): |
2008 | else: | 2026 | self.willparse.add((filename, appends)) |
2009 | self.fromcache.append((filename, appends)) | 2027 | else: |
2010 | self.toparse = self.total - len(self.fromcache) | 2028 | self.fromcache.add((filename, appends)) |
2029 | |||
2030 | self.total = len(self.fromcache) + len(self.willparse) | ||
2031 | self.toparse = len(self.willparse) | ||
2011 | self.progress_chunk = int(max(self.toparse / 100, 1)) | 2032 | self.progress_chunk = int(max(self.toparse / 100, 1)) |
2012 | 2033 | ||
2013 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or | 2034 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or |
2014 | multiprocessing.cpu_count()), len(self.willparse)) | 2035 | multiprocessing.cpu_count()), self.toparse) |
2015 | 2036 | ||
2016 | self.start() | 2037 | self.start() |
2017 | self.haveshutdown = False | 2038 | self.haveshutdown = False |
@@ -2032,7 +2053,7 @@ class CookerParser(object): | |||
2032 | 2053 | ||
2033 | def chunkify(lst,n): | 2054 | def chunkify(lst,n): |
2034 | return [lst[i::n] for i in range(n)] | 2055 | return [lst[i::n] for i in range(n)] |
2035 | self.jobs = chunkify(self.willparse, self.num_processes) | 2056 | self.jobs = chunkify(list(self.willparse), self.num_processes) |
2036 | 2057 | ||
2037 | for i in range(0, self.num_processes): | 2058 | for i in range(0, self.num_processes): |
2038 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) | 2059 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) |
@@ -2095,9 +2116,9 @@ class CookerParser(object): | |||
2095 | print("Processed parsing statistics saved to %s" % (pout)) | 2116 | print("Processed parsing statistics saved to %s" % (pout)) |
2096 | 2117 | ||
2097 | def load_cached(self): | 2118 | def load_cached(self): |
2098 | for filename, appends in self.fromcache: | 2119 | for mc, filename, appends in self.fromcache: |
2099 | cached, infos = self.bb_cache.load(filename, appends) | 2120 | cached, infos = self.bb_cache.load(mc, filename, appends) |
2100 | yield not cached, infos | 2121 | yield not cached, mc, infos |
2101 | 2122 | ||
2102 | def parse_generator(self): | 2123 | def parse_generator(self): |
2103 | while True: | 2124 | while True: |
@@ -2119,7 +2140,7 @@ class CookerParser(object): | |||
2119 | result = [] | 2140 | result = [] |
2120 | parsed = None | 2141 | parsed = None |
2121 | try: | 2142 | try: |
2122 | parsed, result = next(self.results) | 2143 | parsed, mc, result = next(self.results) |
2123 | except StopIteration: | 2144 | except StopIteration: |
2124 | self.shutdown() | 2145 | self.shutdown() |
2125 | return False | 2146 | return False |
@@ -2181,7 +2202,11 @@ class CookerParser(object): | |||
2181 | return True | 2202 | return True |
2182 | 2203 | ||
2183 | def reparse(self, filename): | 2204 | def reparse(self, filename): |
2184 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) | 2205 | to_reparse = set() |
2185 | for vfn, info_array in infos: | 2206 | for mc in self.cooker.multiconfigs: |
2186 | (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) | 2207 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) |
2187 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | 2208 | |
2209 | for mc, filename, appends in to_reparse: | ||
2210 | infos = self.bb_cache.parse(filename, appends) | ||
2211 | for vfn, info_array in infos: | ||
2212 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | ||