diff options
-rw-r--r-- | bitbake/lib/bb/cache.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/command.py | 37 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 139 | ||||
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 11 | ||||
-rw-r--r-- | bitbake/lib/bb/tinfoil.py | 24 | ||||
-rw-r--r-- | bitbake/lib/bblayers/action.py | 22 | ||||
-rw-r--r-- | bitbake/lib/bblayers/query.py | 4 |
7 files changed, 148 insertions, 91 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index d1be83617b..aa5ec5b591 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -623,7 +623,7 @@ class Cache(NoCache): | |||
623 | self.remove(fn) | 623 | self.remove(fn) |
624 | return False | 624 | return False |
625 | 625 | ||
626 | if appends != info_array[0].appends: | 626 | if tuple(appends) != tuple(info_array[0].appends): |
627 | logger.debug(2, "Cache: appends for %s changed", fn) | 627 | logger.debug(2, "Cache: appends for %s changed", fn) |
628 | logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) | 628 | logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) |
629 | self.remove(fn) | 629 | self.remove(fn) |
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py index 6abf38668b..d11907e3ba 100644 --- a/bitbake/lib/bb/command.py +++ b/bitbake/lib/bb/command.py | |||
@@ -232,7 +232,11 @@ class CommandsSync: | |||
232 | 232 | ||
233 | def matchFile(self, command, params): | 233 | def matchFile(self, command, params): |
234 | fMatch = params[0] | 234 | fMatch = params[0] |
235 | return command.cooker.matchFile(fMatch) | 235 | try: |
236 | mc = params[0] | ||
237 | except IndexError: | ||
238 | mc = '' | ||
239 | return command.cooker.matchFile(fMatch, mc) | ||
236 | matchFile.needconfig = False | 240 | matchFile.needconfig = False |
237 | 241 | ||
238 | def getUIHandlerNum(self, command, params): | 242 | def getUIHandlerNum(self, command, params): |
@@ -395,22 +399,38 @@ class CommandsSync: | |||
395 | def getSkippedRecipes(self, command, params): | 399 | def getSkippedRecipes(self, command, params): |
396 | # Return list sorted by reverse priority order | 400 | # Return list sorted by reverse priority order |
397 | import bb.cache | 401 | import bb.cache |
398 | skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), | 402 | def sortkey(x): |
399 | key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) | 403 | vfn, _ = x |
404 | realfn, _, mc = bb.cache.virtualfn2realfn(vfn) | ||
405 | return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn) | ||
406 | |||
407 | skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) | ||
400 | return list(skipdict.items()) | 408 | return list(skipdict.items()) |
401 | getSkippedRecipes.readonly = True | 409 | getSkippedRecipes.readonly = True |
402 | 410 | ||
403 | def getOverlayedRecipes(self, command, params): | 411 | def getOverlayedRecipes(self, command, params): |
404 | return list(command.cooker.collection.overlayed.items()) | 412 | try: |
413 | mc = params[0] | ||
414 | except IndexError: | ||
415 | mc = '' | ||
416 | return list(command.cooker.collections[mc].overlayed.items()) | ||
405 | getOverlayedRecipes.readonly = True | 417 | getOverlayedRecipes.readonly = True |
406 | 418 | ||
407 | def getFileAppends(self, command, params): | 419 | def getFileAppends(self, command, params): |
408 | fn = params[0] | 420 | fn = params[0] |
409 | return command.cooker.collection.get_file_appends(fn) | 421 | try: |
422 | mc = params[1] | ||
423 | except IndexError: | ||
424 | mc = '' | ||
425 | return command.cooker.collections[mc].get_file_appends(fn) | ||
410 | getFileAppends.readonly = True | 426 | getFileAppends.readonly = True |
411 | 427 | ||
412 | def getAllAppends(self, command, params): | 428 | def getAllAppends(self, command, params): |
413 | return command.cooker.collection.bbappends | 429 | try: |
430 | mc = params[0] | ||
431 | except IndexError: | ||
432 | mc = '' | ||
433 | return command.cooker.collections[mc].bbappends | ||
414 | getAllAppends.readonly = True | 434 | getAllAppends.readonly = True |
415 | 435 | ||
416 | def findProviders(self, command, params): | 436 | def findProviders(self, command, params): |
@@ -496,6 +516,7 @@ class CommandsSync: | |||
496 | for the recipe. | 516 | for the recipe. |
497 | """ | 517 | """ |
498 | fn = params[0] | 518 | fn = params[0] |
519 | mc = bb.runqueue.mc_from_tid(fn) | ||
499 | appends = params[1] | 520 | appends = params[1] |
500 | appendlist = params[2] | 521 | appendlist = params[2] |
501 | if len(params) > 3: | 522 | if len(params) > 3: |
@@ -507,7 +528,7 @@ class CommandsSync: | |||
507 | if appendlist is not None: | 528 | if appendlist is not None: |
508 | appendfiles = appendlist | 529 | appendfiles = appendlist |
509 | else: | 530 | else: |
510 | appendfiles = command.cooker.collection.get_file_appends(fn) | 531 | appendfiles = command.cooker.collections[mc].get_file_appends(fn) |
511 | else: | 532 | else: |
512 | appendfiles = [] | 533 | appendfiles = [] |
513 | # We are calling bb.cache locally here rather than on the server, | 534 | # We are calling bb.cache locally here rather than on the server, |
@@ -517,7 +538,7 @@ class CommandsSync: | |||
517 | if config_data: | 538 | if config_data: |
518 | # We have to use a different function here if we're passing in a datastore | 539 | # We have to use a different function here if we're passing in a datastore |
519 | # NOTE: we took a copy above, so we don't do it here again | 540 | # NOTE: we took a copy above, so we don't do it here again |
520 | envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] | 541 | envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] |
521 | else: | 542 | else: |
522 | # Use the standard path | 543 | # Use the standard path |
523 | parser = bb.cache.NoCache(command.cooker.databuilder) | 544 | parser = bb.cache.NoCache(command.cooker.databuilder) |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index e527e23114..8f45233c8d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -525,7 +525,7 @@ class BBCooker: | |||
525 | self.parseConfiguration() | 525 | self.parseConfiguration() |
526 | 526 | ||
527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
528 | fn = self.matchFile(fn) | 528 | fn = self.matchFile(fn, mc) |
529 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 529 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
530 | elif len(pkgs_to_build) == 1: | 530 | elif len(pkgs_to_build) == 1: |
531 | mc = mc_base(pkgs_to_build[0]) | 531 | mc = mc_base(pkgs_to_build[0]) |
@@ -542,7 +542,7 @@ class BBCooker: | |||
542 | if fn: | 542 | if fn: |
543 | try: | 543 | try: |
544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
545 | envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) | 545 | envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn)) |
546 | except Exception as e: | 546 | except Exception as e: |
547 | parselog.exception("Unable to read %s", fn) | 547 | parselog.exception("Unable to read %s", fn) |
548 | raise | 548 | raise |
@@ -929,26 +929,33 @@ class BBCooker: | |||
929 | logger.info("Task dependencies saved to 'task-depends.dot'") | 929 | logger.info("Task dependencies saved to 'task-depends.dot'") |
930 | 930 | ||
931 | def show_appends_with_no_recipes(self): | 931 | def show_appends_with_no_recipes(self): |
932 | appends_without_recipes = {} | ||
932 | # Determine which bbappends haven't been applied | 933 | # Determine which bbappends haven't been applied |
933 | 934 | for mc in self.multiconfigs: | |
934 | # First get list of recipes, including skipped | 935 | # First get list of recipes, including skipped |
935 | recipefns = list(self.recipecaches[''].pkg_fn.keys()) | 936 | recipefns = list(self.recipecaches[mc].pkg_fn.keys()) |
936 | recipefns.extend(self.skiplist.keys()) | 937 | recipefns.extend(self.skiplist.keys()) |
937 | 938 | ||
938 | # Work out list of bbappends that have been applied | 939 | # Work out list of bbappends that have been applied |
939 | applied_appends = [] | 940 | applied_appends = [] |
940 | for fn in recipefns: | 941 | for fn in recipefns: |
941 | applied_appends.extend(self.collection.get_file_appends(fn)) | 942 | applied_appends.extend(self.collections[mc].get_file_appends(fn)) |
942 | 943 | ||
943 | appends_without_recipes = [] | 944 | appends_without_recipes[mc] = [] |
944 | for _, appendfn in self.collection.bbappends: | 945 | for _, appendfn in self.collections[mc].bbappends: |
945 | if not appendfn in applied_appends: | 946 | if not appendfn in applied_appends: |
946 | appends_without_recipes.append(appendfn) | 947 | appends_without_recipes[mc].append(appendfn) |
947 | 948 | ||
948 | if appends_without_recipes: | 949 | msgs = [] |
949 | msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) | 950 | for mc in sorted(appends_without_recipes.keys()): |
950 | warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | 951 | if appends_without_recipes[mc]: |
951 | False) or "no" | 952 | msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', |
953 | '\n '.join(appends_without_recipes[mc]))) | ||
954 | |||
955 | if msgs: | ||
956 | msg = "\n".join(msgs) | ||
957 | warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | ||
958 | False) or "no" | ||
952 | if warn_only.lower() in ("1", "yes", "true"): | 959 | if warn_only.lower() in ("1", "yes", "true"): |
953 | bb.warn(msg) | 960 | bb.warn(msg) |
954 | else: | 961 | else: |
@@ -1249,15 +1256,15 @@ class BBCooker: | |||
1249 | if siggen_cache: | 1256 | if siggen_cache: |
1250 | bb.parse.siggen.checksum_cache.mtime_cache.clear() | 1257 | bb.parse.siggen.checksum_cache.mtime_cache.clear() |
1251 | 1258 | ||
1252 | def matchFiles(self, bf): | 1259 | def matchFiles(self, bf, mc=''): |
1253 | """ | 1260 | """ |
1254 | Find the .bb files which match the expression in 'buildfile'. | 1261 | Find the .bb files which match the expression in 'buildfile'. |
1255 | """ | 1262 | """ |
1256 | if bf.startswith("/") or bf.startswith("../"): | 1263 | if bf.startswith("/") or bf.startswith("../"): |
1257 | bf = os.path.abspath(bf) | 1264 | bf = os.path.abspath(bf) |
1258 | 1265 | ||
1259 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1266 | self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} |
1260 | filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) | 1267 | filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) |
1261 | try: | 1268 | try: |
1262 | os.stat(bf) | 1269 | os.stat(bf) |
1263 | bf = os.path.abspath(bf) | 1270 | bf = os.path.abspath(bf) |
@@ -1270,12 +1277,12 @@ class BBCooker: | |||
1270 | matches.append(f) | 1277 | matches.append(f) |
1271 | return matches | 1278 | return matches |
1272 | 1279 | ||
1273 | def matchFile(self, buildfile): | 1280 | def matchFile(self, buildfile, mc=''): |
1274 | """ | 1281 | """ |
1275 | Find the .bb file which matches the expression in 'buildfile'. | 1282 | Find the .bb file which matches the expression in 'buildfile'. |
1276 | Raise an error if multiple files | 1283 | Raise an error if multiple files |
1277 | """ | 1284 | """ |
1278 | matches = self.matchFiles(buildfile) | 1285 | matches = self.matchFiles(buildfile, mc) |
1279 | if len(matches) != 1: | 1286 | if len(matches) != 1: |
1280 | if matches: | 1287 | if matches: |
1281 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) | 1288 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) |
@@ -1316,14 +1323,14 @@ class BBCooker: | |||
1316 | task = "do_%s" % task | 1323 | task = "do_%s" % task |
1317 | 1324 | ||
1318 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 1325 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
1319 | fn = self.matchFile(fn) | 1326 | fn = self.matchFile(fn, mc) |
1320 | 1327 | ||
1321 | self.buildSetVars() | 1328 | self.buildSetVars() |
1322 | self.reset_mtime_caches() | 1329 | self.reset_mtime_caches() |
1323 | 1330 | ||
1324 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 1331 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
1325 | 1332 | ||
1326 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) | 1333 | infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn)) |
1327 | infos = dict(infos) | 1334 | infos = dict(infos) |
1328 | 1335 | ||
1329 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 1336 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
@@ -1552,14 +1559,24 @@ class BBCooker: | |||
1552 | for dep in self.configuration.extra_assume_provided: | 1559 | for dep in self.configuration.extra_assume_provided: |
1553 | self.recipecaches[mc].ignored_dependencies.add(dep) | 1560 | self.recipecaches[mc].ignored_dependencies.add(dep) |
1554 | 1561 | ||
1555 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1562 | self.collections = {} |
1556 | (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) | 1563 | |
1564 | mcfilelist = {} | ||
1565 | total_masked = 0 | ||
1566 | searchdirs = set() | ||
1567 | for mc in self.multiconfigs: | ||
1568 | self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) | ||
1569 | (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) | ||
1570 | |||
1571 | mcfilelist[mc] = filelist | ||
1572 | total_masked += masked | ||
1573 | searchdirs |= set(search) | ||
1557 | 1574 | ||
1558 | # Add inotify watches for directories searched for bb/bbappend files | 1575 | # Add inotify watches for directories searched for bb/bbappend files |
1559 | for dirent in searchdirs: | 1576 | for dirent in searchdirs: |
1560 | self.add_filewatch([[dirent]], dirs=True) | 1577 | self.add_filewatch([[dirent]], dirs=True) |
1561 | 1578 | ||
1562 | self.parser = CookerParser(self, filelist, masked) | 1579 | self.parser = CookerParser(self, mcfilelist, total_masked) |
1563 | self.parsecache_valid = True | 1580 | self.parsecache_valid = True |
1564 | 1581 | ||
1565 | self.state = state.parsing | 1582 | self.state = state.parsing |
@@ -1571,7 +1588,7 @@ class BBCooker: | |||
1571 | self.show_appends_with_no_recipes() | 1588 | self.show_appends_with_no_recipes() |
1572 | self.handlePrefProviders() | 1589 | self.handlePrefProviders() |
1573 | for mc in self.multiconfigs: | 1590 | for mc in self.multiconfigs: |
1574 | self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) | 1591 | self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) |
1575 | self.state = state.running | 1592 | self.state = state.running |
1576 | 1593 | ||
1577 | # Send an event listing all stamps reachable after parsing | 1594 | # Send an event listing all stamps reachable after parsing |
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event): | |||
1679 | 1696 | ||
1680 | 1697 | ||
1681 | class CookerCollectFiles(object): | 1698 | class CookerCollectFiles(object): |
1682 | def __init__(self, priorities): | 1699 | def __init__(self, priorities, mc=''): |
1700 | self.mc = mc | ||
1683 | self.bbappends = [] | 1701 | self.bbappends = [] |
1684 | # Priorities is a list of tupples, with the second element as the pattern. | 1702 | # Priorities is a list of tupples, with the second element as the pattern. |
1685 | # We need to sort the list with the longest pattern first, and so on to | 1703 | # We need to sort the list with the longest pattern first, and so on to |
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object): | |||
1846 | (bbappend, filename) = b | 1864 | (bbappend, filename) = b |
1847 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): | 1865 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): |
1848 | filelist.append(filename) | 1866 | filelist.append(filename) |
1849 | return filelist | 1867 | return tuple(filelist) |
1850 | 1868 | ||
1851 | def collection_priorities(self, pkgfns, d): | 1869 | def collection_priorities(self, pkgfns, d): |
1852 | 1870 | ||
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object): | |||
1882 | for collection, pattern, regex, _ in self.bbfile_config_priorities: | 1900 | for collection, pattern, regex, _ in self.bbfile_config_priorities: |
1883 | if regex in unmatched: | 1901 | if regex in unmatched: |
1884 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': | 1902 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': |
1885 | collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) | 1903 | collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', |
1904 | collection, pattern)) | ||
1886 | 1905 | ||
1887 | return priorities | 1906 | return priorities |
1888 | 1907 | ||
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process): | |||
1978 | bb.event.LogHandler.filter = origfilter | 1997 | bb.event.LogHandler.filter = origfilter |
1979 | 1998 | ||
1980 | class CookerParser(object): | 1999 | class CookerParser(object): |
1981 | def __init__(self, cooker, filelist, masked): | 2000 | def __init__(self, cooker, mcfilelist, masked): |
1982 | self.filelist = filelist | 2001 | self.mcfilelist = mcfilelist |
1983 | self.cooker = cooker | 2002 | self.cooker = cooker |
1984 | self.cfgdata = cooker.data | 2003 | self.cfgdata = cooker.data |
1985 | self.cfghash = cooker.data_hash | 2004 | self.cfghash = cooker.data_hash |
@@ -1993,25 +2012,27 @@ class CookerParser(object): | |||
1993 | 2012 | ||
1994 | self.skipped = 0 | 2013 | self.skipped = 0 |
1995 | self.virtuals = 0 | 2014 | self.virtuals = 0 |
1996 | self.total = len(filelist) | ||
1997 | 2015 | ||
1998 | self.current = 0 | 2016 | self.current = 0 |
1999 | self.process_names = [] | 2017 | self.process_names = [] |
2000 | 2018 | ||
2001 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) | 2019 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) |
2002 | self.fromcache = [] | 2020 | self.fromcache = set() |
2003 | self.willparse = [] | 2021 | self.willparse = set() |
2004 | for filename in self.filelist: | 2022 | for mc in self.cooker.multiconfigs: |
2005 | appends = self.cooker.collection.get_file_appends(filename) | 2023 | for filename in self.mcfilelist[mc]: |
2006 | if not self.bb_cache.cacheValid(filename, appends): | 2024 | appends = self.cooker.collections[mc].get_file_appends(filename) |
2007 | self.willparse.append((filename, appends)) | 2025 | if not self.bb_cache.cacheValid(filename, appends): |
2008 | else: | 2026 | self.willparse.add((filename, appends)) |
2009 | self.fromcache.append((filename, appends)) | 2027 | else: |
2010 | self.toparse = self.total - len(self.fromcache) | 2028 | self.fromcache.add((filename, appends)) |
2029 | |||
2030 | self.total = len(self.fromcache) + len(self.willparse) | ||
2031 | self.toparse = len(self.willparse) | ||
2011 | self.progress_chunk = int(max(self.toparse / 100, 1)) | 2032 | self.progress_chunk = int(max(self.toparse / 100, 1)) |
2012 | 2033 | ||
2013 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or | 2034 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or |
2014 | multiprocessing.cpu_count()), len(self.willparse)) | 2035 | multiprocessing.cpu_count()), self.toparse) |
2015 | 2036 | ||
2016 | self.start() | 2037 | self.start() |
2017 | self.haveshutdown = False | 2038 | self.haveshutdown = False |
@@ -2032,7 +2053,7 @@ class CookerParser(object): | |||
2032 | 2053 | ||
2033 | def chunkify(lst,n): | 2054 | def chunkify(lst,n): |
2034 | return [lst[i::n] for i in range(n)] | 2055 | return [lst[i::n] for i in range(n)] |
2035 | self.jobs = chunkify(self.willparse, self.num_processes) | 2056 | self.jobs = chunkify(list(self.willparse), self.num_processes) |
2036 | 2057 | ||
2037 | for i in range(0, self.num_processes): | 2058 | for i in range(0, self.num_processes): |
2038 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) | 2059 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) |
@@ -2095,9 +2116,9 @@ class CookerParser(object): | |||
2095 | print("Processed parsing statistics saved to %s" % (pout)) | 2116 | print("Processed parsing statistics saved to %s" % (pout)) |
2096 | 2117 | ||
2097 | def load_cached(self): | 2118 | def load_cached(self): |
2098 | for filename, appends in self.fromcache: | 2119 | for mc, filename, appends in self.fromcache: |
2099 | cached, infos = self.bb_cache.load(filename, appends) | 2120 | cached, infos = self.bb_cache.load(mc, filename, appends) |
2100 | yield not cached, infos | 2121 | yield not cached, mc, infos |
2101 | 2122 | ||
2102 | def parse_generator(self): | 2123 | def parse_generator(self): |
2103 | while True: | 2124 | while True: |
@@ -2119,7 +2140,7 @@ class CookerParser(object): | |||
2119 | result = [] | 2140 | result = [] |
2120 | parsed = None | 2141 | parsed = None |
2121 | try: | 2142 | try: |
2122 | parsed, result = next(self.results) | 2143 | parsed, mc, result = next(self.results) |
2123 | except StopIteration: | 2144 | except StopIteration: |
2124 | self.shutdown() | 2145 | self.shutdown() |
2125 | return False | 2146 | return False |
@@ -2181,7 +2202,11 @@ class CookerParser(object): | |||
2181 | return True | 2202 | return True |
2182 | 2203 | ||
2183 | def reparse(self, filename): | 2204 | def reparse(self, filename): |
2184 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) | 2205 | to_reparse = set() |
2185 | for vfn, info_array in infos: | 2206 | for mc in self.cooker.multiconfigs: |
2186 | (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) | 2207 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) |
2187 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | 2208 | |
2209 | for mc, filename, appends in to_reparse: | ||
2210 | infos = self.bb_cache.parse(filename, appends) | ||
2211 | for vfn, info_array in infos: | ||
2212 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | ||
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 16f076f3b1..3d54c2b88a 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1557,7 +1557,8 @@ class RunQueue: | |||
1557 | 1557 | ||
1558 | def rq_dump_sigfn(self, fn, options): | 1558 | def rq_dump_sigfn(self, fn, options): |
1559 | bb_cache = bb.cache.NoCache(self.cooker.databuilder) | 1559 | bb_cache = bb.cache.NoCache(self.cooker.databuilder) |
1560 | the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) | 1560 | mc = bb.runqueue.mc_from_tid(fn) |
1561 | the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) | ||
1561 | siggen = bb.parse.siggen | 1562 | siggen = bb.parse.siggen |
1562 | dataCaches = self.rqdata.dataCaches | 1563 | dataCaches = self.rqdata.dataCaches |
1563 | siggen.dump_sigfn(fn, dataCaches, options) | 1564 | siggen.dump_sigfn(fn, dataCaches, options) |
@@ -2042,10 +2043,10 @@ class RunQueueExecute: | |||
2042 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | 2043 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: |
2043 | if not mc in self.rq.fakeworker: | 2044 | if not mc in self.rq.fakeworker: |
2044 | self.rq.start_fakeworker(self, mc) | 2045 | self.rq.start_fakeworker(self, mc) |
2045 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | 2046 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") |
2046 | self.rq.fakeworker[mc].process.stdin.flush() | 2047 | self.rq.fakeworker[mc].process.stdin.flush() |
2047 | else: | 2048 | else: |
2048 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | 2049 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") |
2049 | self.rq.worker[mc].process.stdin.flush() | 2050 | self.rq.worker[mc].process.stdin.flush() |
2050 | 2051 | ||
2051 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | 2052 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) |
@@ -2129,10 +2130,10 @@ class RunQueueExecute: | |||
2129 | self.rq.state = runQueueFailed | 2130 | self.rq.state = runQueueFailed |
2130 | self.stats.taskFailed() | 2131 | self.stats.taskFailed() |
2131 | return True | 2132 | return True |
2132 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") | 2133 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") |
2133 | self.rq.fakeworker[mc].process.stdin.flush() | 2134 | self.rq.fakeworker[mc].process.stdin.flush() |
2134 | else: | 2135 | else: |
2135 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") | 2136 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") |
2136 | self.rq.worker[mc].process.stdin.flush() | 2137 | self.rq.worker[mc].process.stdin.flush() |
2137 | 2138 | ||
2138 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | 2139 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) |
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index 8c9b6b8ca5..dccbe0ebb5 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: | |||
117 | 117 | ||
118 | class TinfoilCookerCollectionAdapter: | 118 | class TinfoilCookerCollectionAdapter: |
119 | """ cooker.collection adapter """ | 119 | """ cooker.collection adapter """ |
120 | def __init__(self, tinfoil): | 120 | def __init__(self, tinfoil, mc=''): |
121 | self.tinfoil = tinfoil | 121 | self.tinfoil = tinfoil |
122 | self.mc = mc | ||
122 | def get_file_appends(self, fn): | 123 | def get_file_appends(self, fn): |
123 | return self.tinfoil.get_file_appends(fn) | 124 | return self.tinfoil.get_file_appends(fn, self.mc) |
124 | def __getattr__(self, name): | 125 | def __getattr__(self, name): |
125 | if name == 'overlayed': | 126 | if name == 'overlayed': |
126 | return self.tinfoil.get_overlayed_recipes() | 127 | return self.tinfoil.get_overlayed_recipes(self.mc) |
127 | elif name == 'bbappends': | 128 | elif name == 'bbappends': |
128 | return self.tinfoil.run_command('getAllAppends') | 129 | return self.tinfoil.run_command('getAllAppends', self.mc) |
129 | else: | 130 | else: |
130 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) | 131 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) |
131 | 132 | ||
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: | |||
185 | 186 | ||
186 | def __init__(self, tinfoil): | 187 | def __init__(self, tinfoil): |
187 | self.tinfoil = tinfoil | 188 | self.tinfoil = tinfoil |
188 | self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) | 189 | self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() |
190 | self.collections = {} | ||
189 | self.recipecaches = {} | 191 | self.recipecaches = {} |
190 | self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) | 192 | for mc in self.multiconfigs: |
191 | for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): | 193 | self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) |
192 | self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) | 194 | self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) |
193 | self._cache = {} | 195 | self._cache = {} |
194 | def __getattr__(self, name): | 196 | def __getattr__(self, name): |
@@ -492,11 +494,11 @@ class Tinfoil: | |||
492 | raise Exception('Not connected to server (did you call .prepare()?)') | 494 | raise Exception('Not connected to server (did you call .prepare()?)') |
493 | return self.server_connection.events.waitEvent(timeout) | 495 | return self.server_connection.events.waitEvent(timeout) |
494 | 496 | ||
495 | def get_overlayed_recipes(self): | 497 | def get_overlayed_recipes(self, mc=''): |
496 | """ | 498 | """ |
497 | Find recipes which are overlayed (i.e. where recipes exist in multiple layers) | 499 | Find recipes which are overlayed (i.e. where recipes exist in multiple layers) |
498 | """ | 500 | """ |
499 | return defaultdict(list, self.run_command('getOverlayedRecipes')) | 501 | return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) |
500 | 502 | ||
501 | def get_skipped_recipes(self): | 503 | def get_skipped_recipes(self): |
502 | """ | 504 | """ |
@@ -534,11 +536,11 @@ class Tinfoil: | |||
534 | raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) | 536 | raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) |
535 | return best[3] | 537 | return best[3] |
536 | 538 | ||
537 | def get_file_appends(self, fn): | 539 | def get_file_appends(self, fn, mc=''): |
538 | """ | 540 | """ |
539 | Find the bbappends for a recipe file | 541 | Find the bbappends for a recipe file |
540 | """ | 542 | """ |
541 | return self.run_command('getFileAppends', fn) | 543 | return self.run_command('getFileAppends', fn, mc) |
542 | 544 | ||
543 | def all_recipes(self, mc='', sort=True): | 545 | def all_recipes(self, mc='', sort=True): |
544 | """ | 546 | """ |
diff --git a/bitbake/lib/bblayers/action.py b/bitbake/lib/bblayers/action.py index d6459d6617..5b78195ad4 100644 --- a/bitbake/lib/bblayers/action.py +++ b/bitbake/lib/bblayers/action.py | |||
@@ -143,11 +143,12 @@ build results (as the layer priority order has effectively changed). | |||
143 | 143 | ||
144 | applied_appends = [] | 144 | applied_appends = [] |
145 | for layer in layers: | 145 | for layer in layers: |
146 | overlayed = [] | 146 | overlayed = set() |
147 | for f in self.tinfoil.cooker.collection.overlayed.keys(): | 147 | for mc in self.tinfoil.cooker.multiconfigs: |
148 | for of in self.tinfoil.cooker.collection.overlayed[f]: | 148 | for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): |
149 | if of.startswith(layer): | 149 | for of in self.tinfoil.cooker.collections[mc].overlayed[f]: |
150 | overlayed.append(of) | 150 | if of.startswith(layer): |
151 | overlayed.add(of) | ||
151 | 152 | ||
152 | logger.plain('Copying files from %s...' % layer ) | 153 | logger.plain('Copying files from %s...' % layer ) |
153 | for root, dirs, files in os.walk(layer): | 154 | for root, dirs, files in os.walk(layer): |
@@ -174,14 +175,21 @@ build results (as the layer priority order has effectively changed). | |||
174 | logger.warning('Overwriting file %s', fdest) | 175 | logger.warning('Overwriting file %s', fdest) |
175 | bb.utils.copyfile(f1full, fdest) | 176 | bb.utils.copyfile(f1full, fdest) |
176 | if ext == '.bb': | 177 | if ext == '.bb': |
177 | for append in self.tinfoil.cooker.collection.get_file_appends(f1full): | 178 | appends = set() |
179 | for mc in self.tinfoil.cooker.multiconfigs: | ||
180 | appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) | ||
181 | for append in appends: | ||
178 | if layer_path_match(append): | 182 | if layer_path_match(append): |
179 | logger.plain(' Applying append %s to %s' % (append, fdest)) | 183 | logger.plain(' Applying append %s to %s' % (append, fdest)) |
180 | self.apply_append(append, fdest) | 184 | self.apply_append(append, fdest) |
181 | applied_appends.append(append) | 185 | applied_appends.append(append) |
182 | 186 | ||
183 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes | 187 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes |
184 | for b in self.tinfoil.cooker.collection.bbappends: | 188 | bbappends = set() |
189 | for mc in self.tinfoil.cooker.multiconfigs: | ||
190 | bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) | ||
191 | |||
192 | for b in bbappends: | ||
185 | (recipename, appendname) = b | 193 | (recipename, appendname) = b |
186 | if appendname not in applied_appends: | 194 | if appendname not in applied_appends: |
187 | first_append = None | 195 | first_append = None |
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py index e2cc310532..ee2db0efed 100644 --- a/bitbake/lib/bblayers/query.py +++ b/bitbake/lib/bblayers/query.py | |||
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. | |||
320 | def get_appends_for_files(self, filenames): | 320 | def get_appends_for_files(self, filenames): |
321 | appended, notappended = [], [] | 321 | appended, notappended = [], [] |
322 | for filename in filenames: | 322 | for filename in filenames: |
323 | _, cls, _ = bb.cache.virtualfn2realfn(filename) | 323 | _, cls, mc = bb.cache.virtualfn2realfn(filename) |
324 | if cls: | 324 | if cls: |
325 | continue | 325 | continue |
326 | 326 | ||
327 | basename = os.path.basename(filename) | 327 | basename = os.path.basename(filename) |
328 | appends = self.tinfoil.cooker.collection.get_file_appends(basename) | 328 | appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) |
329 | if appends: | 329 | if appends: |
330 | appended.append((basename, list(appends))) | 330 | appended.append((basename, list(appends))) |
331 | else: | 331 | else: |