diff options
| -rw-r--r-- | bitbake/lib/bb/cache.py | 2 | ||||
| -rw-r--r-- | bitbake/lib/bb/command.py | 37 | ||||
| -rw-r--r-- | bitbake/lib/bb/cooker.py | 139 | ||||
| -rw-r--r-- | bitbake/lib/bb/runqueue.py | 11 | ||||
| -rw-r--r-- | bitbake/lib/bb/tinfoil.py | 24 | ||||
| -rw-r--r-- | bitbake/lib/bblayers/action.py | 22 | ||||
| -rw-r--r-- | bitbake/lib/bblayers/query.py | 4 |
7 files changed, 148 insertions, 91 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index d1be83617b..aa5ec5b591 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
| @@ -623,7 +623,7 @@ class Cache(NoCache): | |||
| 623 | self.remove(fn) | 623 | self.remove(fn) |
| 624 | return False | 624 | return False |
| 625 | 625 | ||
| 626 | if appends != info_array[0].appends: | 626 | if tuple(appends) != tuple(info_array[0].appends): |
| 627 | logger.debug(2, "Cache: appends for %s changed", fn) | 627 | logger.debug(2, "Cache: appends for %s changed", fn) |
| 628 | logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) | 628 | logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) |
| 629 | self.remove(fn) | 629 | self.remove(fn) |
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py index 6abf38668b..d11907e3ba 100644 --- a/bitbake/lib/bb/command.py +++ b/bitbake/lib/bb/command.py | |||
| @@ -232,7 +232,11 @@ class CommandsSync: | |||
| 232 | 232 | ||
| 233 | def matchFile(self, command, params): | 233 | def matchFile(self, command, params): |
| 234 | fMatch = params[0] | 234 | fMatch = params[0] |
| 235 | return command.cooker.matchFile(fMatch) | 235 | try: |
| 236 | mc = params[0] | ||
| 237 | except IndexError: | ||
| 238 | mc = '' | ||
| 239 | return command.cooker.matchFile(fMatch, mc) | ||
| 236 | matchFile.needconfig = False | 240 | matchFile.needconfig = False |
| 237 | 241 | ||
| 238 | def getUIHandlerNum(self, command, params): | 242 | def getUIHandlerNum(self, command, params): |
| @@ -395,22 +399,38 @@ class CommandsSync: | |||
| 395 | def getSkippedRecipes(self, command, params): | 399 | def getSkippedRecipes(self, command, params): |
| 396 | # Return list sorted by reverse priority order | 400 | # Return list sorted by reverse priority order |
| 397 | import bb.cache | 401 | import bb.cache |
| 398 | skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), | 402 | def sortkey(x): |
| 399 | key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) | 403 | vfn, _ = x |
| 404 | realfn, _, mc = bb.cache.virtualfn2realfn(vfn) | ||
| 405 | return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn) | ||
| 406 | |||
| 407 | skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) | ||
| 400 | return list(skipdict.items()) | 408 | return list(skipdict.items()) |
| 401 | getSkippedRecipes.readonly = True | 409 | getSkippedRecipes.readonly = True |
| 402 | 410 | ||
| 403 | def getOverlayedRecipes(self, command, params): | 411 | def getOverlayedRecipes(self, command, params): |
| 404 | return list(command.cooker.collection.overlayed.items()) | 412 | try: |
| 413 | mc = params[0] | ||
| 414 | except IndexError: | ||
| 415 | mc = '' | ||
| 416 | return list(command.cooker.collections[mc].overlayed.items()) | ||
| 405 | getOverlayedRecipes.readonly = True | 417 | getOverlayedRecipes.readonly = True |
| 406 | 418 | ||
| 407 | def getFileAppends(self, command, params): | 419 | def getFileAppends(self, command, params): |
| 408 | fn = params[0] | 420 | fn = params[0] |
| 409 | return command.cooker.collection.get_file_appends(fn) | 421 | try: |
| 422 | mc = params[1] | ||
| 423 | except IndexError: | ||
| 424 | mc = '' | ||
| 425 | return command.cooker.collections[mc].get_file_appends(fn) | ||
| 410 | getFileAppends.readonly = True | 426 | getFileAppends.readonly = True |
| 411 | 427 | ||
| 412 | def getAllAppends(self, command, params): | 428 | def getAllAppends(self, command, params): |
| 413 | return command.cooker.collection.bbappends | 429 | try: |
| 430 | mc = params[0] | ||
| 431 | except IndexError: | ||
| 432 | mc = '' | ||
| 433 | return command.cooker.collections[mc].bbappends | ||
| 414 | getAllAppends.readonly = True | 434 | getAllAppends.readonly = True |
| 415 | 435 | ||
| 416 | def findProviders(self, command, params): | 436 | def findProviders(self, command, params): |
| @@ -496,6 +516,7 @@ class CommandsSync: | |||
| 496 | for the recipe. | 516 | for the recipe. |
| 497 | """ | 517 | """ |
| 498 | fn = params[0] | 518 | fn = params[0] |
| 519 | mc = bb.runqueue.mc_from_tid(fn) | ||
| 499 | appends = params[1] | 520 | appends = params[1] |
| 500 | appendlist = params[2] | 521 | appendlist = params[2] |
| 501 | if len(params) > 3: | 522 | if len(params) > 3: |
| @@ -507,7 +528,7 @@ class CommandsSync: | |||
| 507 | if appendlist is not None: | 528 | if appendlist is not None: |
| 508 | appendfiles = appendlist | 529 | appendfiles = appendlist |
| 509 | else: | 530 | else: |
| 510 | appendfiles = command.cooker.collection.get_file_appends(fn) | 531 | appendfiles = command.cooker.collections[mc].get_file_appends(fn) |
| 511 | else: | 532 | else: |
| 512 | appendfiles = [] | 533 | appendfiles = [] |
| 513 | # We are calling bb.cache locally here rather than on the server, | 534 | # We are calling bb.cache locally here rather than on the server, |
| @@ -517,7 +538,7 @@ class CommandsSync: | |||
| 517 | if config_data: | 538 | if config_data: |
| 518 | # We have to use a different function here if we're passing in a datastore | 539 | # We have to use a different function here if we're passing in a datastore |
| 519 | # NOTE: we took a copy above, so we don't do it here again | 540 | # NOTE: we took a copy above, so we don't do it here again |
| 520 | envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] | 541 | envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] |
| 521 | else: | 542 | else: |
| 522 | # Use the standard path | 543 | # Use the standard path |
| 523 | parser = bb.cache.NoCache(command.cooker.databuilder) | 544 | parser = bb.cache.NoCache(command.cooker.databuilder) |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index e527e23114..8f45233c8d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
| @@ -525,7 +525,7 @@ class BBCooker: | |||
| 525 | self.parseConfiguration() | 525 | self.parseConfiguration() |
| 526 | 526 | ||
| 527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 527 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
| 528 | fn = self.matchFile(fn) | 528 | fn = self.matchFile(fn, mc) |
| 529 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 529 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
| 530 | elif len(pkgs_to_build) == 1: | 530 | elif len(pkgs_to_build) == 1: |
| 531 | mc = mc_base(pkgs_to_build[0]) | 531 | mc = mc_base(pkgs_to_build[0]) |
| @@ -542,7 +542,7 @@ class BBCooker: | |||
| 542 | if fn: | 542 | if fn: |
| 543 | try: | 543 | try: |
| 544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 544 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
| 545 | envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) | 545 | envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn)) |
| 546 | except Exception as e: | 546 | except Exception as e: |
| 547 | parselog.exception("Unable to read %s", fn) | 547 | parselog.exception("Unable to read %s", fn) |
| 548 | raise | 548 | raise |
| @@ -929,26 +929,33 @@ class BBCooker: | |||
| 929 | logger.info("Task dependencies saved to 'task-depends.dot'") | 929 | logger.info("Task dependencies saved to 'task-depends.dot'") |
| 930 | 930 | ||
| 931 | def show_appends_with_no_recipes(self): | 931 | def show_appends_with_no_recipes(self): |
| 932 | appends_without_recipes = {} | ||
| 932 | # Determine which bbappends haven't been applied | 933 | # Determine which bbappends haven't been applied |
| 933 | 934 | for mc in self.multiconfigs: | |
| 934 | # First get list of recipes, including skipped | 935 | # First get list of recipes, including skipped |
| 935 | recipefns = list(self.recipecaches[''].pkg_fn.keys()) | 936 | recipefns = list(self.recipecaches[mc].pkg_fn.keys()) |
| 936 | recipefns.extend(self.skiplist.keys()) | 937 | recipefns.extend(self.skiplist.keys()) |
| 937 | 938 | ||
| 938 | # Work out list of bbappends that have been applied | 939 | # Work out list of bbappends that have been applied |
| 939 | applied_appends = [] | 940 | applied_appends = [] |
| 940 | for fn in recipefns: | 941 | for fn in recipefns: |
| 941 | applied_appends.extend(self.collection.get_file_appends(fn)) | 942 | applied_appends.extend(self.collections[mc].get_file_appends(fn)) |
| 942 | 943 | ||
| 943 | appends_without_recipes = [] | 944 | appends_without_recipes[mc] = [] |
| 944 | for _, appendfn in self.collection.bbappends: | 945 | for _, appendfn in self.collections[mc].bbappends: |
| 945 | if not appendfn in applied_appends: | 946 | if not appendfn in applied_appends: |
| 946 | appends_without_recipes.append(appendfn) | 947 | appends_without_recipes[mc].append(appendfn) |
| 947 | 948 | ||
| 948 | if appends_without_recipes: | 949 | msgs = [] |
| 949 | msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) | 950 | for mc in sorted(appends_without_recipes.keys()): |
| 950 | warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | 951 | if appends_without_recipes[mc]: |
| 951 | False) or "no" | 952 | msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', |
| 953 | '\n '.join(appends_without_recipes[mc]))) | ||
| 954 | |||
| 955 | if msgs: | ||
| 956 | msg = "\n".join(msgs) | ||
| 957 | warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ | ||
| 958 | False) or "no" | ||
| 952 | if warn_only.lower() in ("1", "yes", "true"): | 959 | if warn_only.lower() in ("1", "yes", "true"): |
| 953 | bb.warn(msg) | 960 | bb.warn(msg) |
| 954 | else: | 961 | else: |
| @@ -1249,15 +1256,15 @@ class BBCooker: | |||
| 1249 | if siggen_cache: | 1256 | if siggen_cache: |
| 1250 | bb.parse.siggen.checksum_cache.mtime_cache.clear() | 1257 | bb.parse.siggen.checksum_cache.mtime_cache.clear() |
| 1251 | 1258 | ||
| 1252 | def matchFiles(self, bf): | 1259 | def matchFiles(self, bf, mc=''): |
| 1253 | """ | 1260 | """ |
| 1254 | Find the .bb files which match the expression in 'buildfile'. | 1261 | Find the .bb files which match the expression in 'buildfile'. |
| 1255 | """ | 1262 | """ |
| 1256 | if bf.startswith("/") or bf.startswith("../"): | 1263 | if bf.startswith("/") or bf.startswith("../"): |
| 1257 | bf = os.path.abspath(bf) | 1264 | bf = os.path.abspath(bf) |
| 1258 | 1265 | ||
| 1259 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1266 | self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} |
| 1260 | filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) | 1267 | filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) |
| 1261 | try: | 1268 | try: |
| 1262 | os.stat(bf) | 1269 | os.stat(bf) |
| 1263 | bf = os.path.abspath(bf) | 1270 | bf = os.path.abspath(bf) |
| @@ -1270,12 +1277,12 @@ class BBCooker: | |||
| 1270 | matches.append(f) | 1277 | matches.append(f) |
| 1271 | return matches | 1278 | return matches |
| 1272 | 1279 | ||
| 1273 | def matchFile(self, buildfile): | 1280 | def matchFile(self, buildfile, mc=''): |
| 1274 | """ | 1281 | """ |
| 1275 | Find the .bb file which matches the expression in 'buildfile'. | 1282 | Find the .bb file which matches the expression in 'buildfile'. |
| 1276 | Raise an error if multiple files | 1283 | Raise an error if multiple files |
| 1277 | """ | 1284 | """ |
| 1278 | matches = self.matchFiles(buildfile) | 1285 | matches = self.matchFiles(buildfile, mc) |
| 1279 | if len(matches) != 1: | 1286 | if len(matches) != 1: |
| 1280 | if matches: | 1287 | if matches: |
| 1281 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) | 1288 | msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) |
| @@ -1316,14 +1323,14 @@ class BBCooker: | |||
| 1316 | task = "do_%s" % task | 1323 | task = "do_%s" % task |
| 1317 | 1324 | ||
| 1318 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) | 1325 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
| 1319 | fn = self.matchFile(fn) | 1326 | fn = self.matchFile(fn, mc) |
| 1320 | 1327 | ||
| 1321 | self.buildSetVars() | 1328 | self.buildSetVars() |
| 1322 | self.reset_mtime_caches() | 1329 | self.reset_mtime_caches() |
| 1323 | 1330 | ||
| 1324 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) | 1331 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
| 1325 | 1332 | ||
| 1326 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) | 1333 | infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn)) |
| 1327 | infos = dict(infos) | 1334 | infos = dict(infos) |
| 1328 | 1335 | ||
| 1329 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 1336 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
| @@ -1552,14 +1559,24 @@ class BBCooker: | |||
| 1552 | for dep in self.configuration.extra_assume_provided: | 1559 | for dep in self.configuration.extra_assume_provided: |
| 1553 | self.recipecaches[mc].ignored_dependencies.add(dep) | 1560 | self.recipecaches[mc].ignored_dependencies.add(dep) |
| 1554 | 1561 | ||
| 1555 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) | 1562 | self.collections = {} |
| 1556 | (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) | 1563 | |
| 1564 | mcfilelist = {} | ||
| 1565 | total_masked = 0 | ||
| 1566 | searchdirs = set() | ||
| 1567 | for mc in self.multiconfigs: | ||
| 1568 | self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) | ||
| 1569 | (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) | ||
| 1570 | |||
| 1571 | mcfilelist[mc] = filelist | ||
| 1572 | total_masked += masked | ||
| 1573 | searchdirs |= set(search) | ||
| 1557 | 1574 | ||
| 1558 | # Add inotify watches for directories searched for bb/bbappend files | 1575 | # Add inotify watches for directories searched for bb/bbappend files |
| 1559 | for dirent in searchdirs: | 1576 | for dirent in searchdirs: |
| 1560 | self.add_filewatch([[dirent]], dirs=True) | 1577 | self.add_filewatch([[dirent]], dirs=True) |
| 1561 | 1578 | ||
| 1562 | self.parser = CookerParser(self, filelist, masked) | 1579 | self.parser = CookerParser(self, mcfilelist, total_masked) |
| 1563 | self.parsecache_valid = True | 1580 | self.parsecache_valid = True |
| 1564 | 1581 | ||
| 1565 | self.state = state.parsing | 1582 | self.state = state.parsing |
| @@ -1571,7 +1588,7 @@ class BBCooker: | |||
| 1571 | self.show_appends_with_no_recipes() | 1588 | self.show_appends_with_no_recipes() |
| 1572 | self.handlePrefProviders() | 1589 | self.handlePrefProviders() |
| 1573 | for mc in self.multiconfigs: | 1590 | for mc in self.multiconfigs: |
| 1574 | self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) | 1591 | self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) |
| 1575 | self.state = state.running | 1592 | self.state = state.running |
| 1576 | 1593 | ||
| 1577 | # Send an event listing all stamps reachable after parsing | 1594 | # Send an event listing all stamps reachable after parsing |
| @@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event): | |||
| 1679 | 1696 | ||
| 1680 | 1697 | ||
| 1681 | class CookerCollectFiles(object): | 1698 | class CookerCollectFiles(object): |
| 1682 | def __init__(self, priorities): | 1699 | def __init__(self, priorities, mc=''): |
| 1700 | self.mc = mc | ||
| 1683 | self.bbappends = [] | 1701 | self.bbappends = [] |
| 1684 | # Priorities is a list of tupples, with the second element as the pattern. | 1702 | # Priorities is a list of tupples, with the second element as the pattern. |
| 1685 | # We need to sort the list with the longest pattern first, and so on to | 1703 | # We need to sort the list with the longest pattern first, and so on to |
| @@ -1846,7 +1864,7 @@ class CookerCollectFiles(object): | |||
| 1846 | (bbappend, filename) = b | 1864 | (bbappend, filename) = b |
| 1847 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): | 1865 | if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): |
| 1848 | filelist.append(filename) | 1866 | filelist.append(filename) |
| 1849 | return filelist | 1867 | return tuple(filelist) |
| 1850 | 1868 | ||
| 1851 | def collection_priorities(self, pkgfns, d): | 1869 | def collection_priorities(self, pkgfns, d): |
| 1852 | 1870 | ||
| @@ -1882,7 +1900,8 @@ class CookerCollectFiles(object): | |||
| 1882 | for collection, pattern, regex, _ in self.bbfile_config_priorities: | 1900 | for collection, pattern, regex, _ in self.bbfile_config_priorities: |
| 1883 | if regex in unmatched: | 1901 | if regex in unmatched: |
| 1884 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': | 1902 | if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': |
| 1885 | collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) | 1903 | collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', |
| 1904 | collection, pattern)) | ||
| 1886 | 1905 | ||
| 1887 | return priorities | 1906 | return priorities |
| 1888 | 1907 | ||
| @@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process): | |||
| 1978 | bb.event.LogHandler.filter = origfilter | 1997 | bb.event.LogHandler.filter = origfilter |
| 1979 | 1998 | ||
| 1980 | class CookerParser(object): | 1999 | class CookerParser(object): |
| 1981 | def __init__(self, cooker, filelist, masked): | 2000 | def __init__(self, cooker, mcfilelist, masked): |
| 1982 | self.filelist = filelist | 2001 | self.mcfilelist = mcfilelist |
| 1983 | self.cooker = cooker | 2002 | self.cooker = cooker |
| 1984 | self.cfgdata = cooker.data | 2003 | self.cfgdata = cooker.data |
| 1985 | self.cfghash = cooker.data_hash | 2004 | self.cfghash = cooker.data_hash |
| @@ -1993,25 +2012,27 @@ class CookerParser(object): | |||
| 1993 | 2012 | ||
| 1994 | self.skipped = 0 | 2013 | self.skipped = 0 |
| 1995 | self.virtuals = 0 | 2014 | self.virtuals = 0 |
| 1996 | self.total = len(filelist) | ||
| 1997 | 2015 | ||
| 1998 | self.current = 0 | 2016 | self.current = 0 |
| 1999 | self.process_names = [] | 2017 | self.process_names = [] |
| 2000 | 2018 | ||
| 2001 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) | 2019 | self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) |
| 2002 | self.fromcache = [] | 2020 | self.fromcache = set() |
| 2003 | self.willparse = [] | 2021 | self.willparse = set() |
| 2004 | for filename in self.filelist: | 2022 | for mc in self.cooker.multiconfigs: |
| 2005 | appends = self.cooker.collection.get_file_appends(filename) | 2023 | for filename in self.mcfilelist[mc]: |
| 2006 | if not self.bb_cache.cacheValid(filename, appends): | 2024 | appends = self.cooker.collections[mc].get_file_appends(filename) |
| 2007 | self.willparse.append((filename, appends)) | 2025 | if not self.bb_cache.cacheValid(filename, appends): |
| 2008 | else: | 2026 | self.willparse.add((filename, appends)) |
| 2009 | self.fromcache.append((filename, appends)) | 2027 | else: |
| 2010 | self.toparse = self.total - len(self.fromcache) | 2028 | self.fromcache.add((filename, appends)) |
| 2029 | |||
| 2030 | self.total = len(self.fromcache) + len(self.willparse) | ||
| 2031 | self.toparse = len(self.willparse) | ||
| 2011 | self.progress_chunk = int(max(self.toparse / 100, 1)) | 2032 | self.progress_chunk = int(max(self.toparse / 100, 1)) |
| 2012 | 2033 | ||
| 2013 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or | 2034 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or |
| 2014 | multiprocessing.cpu_count()), len(self.willparse)) | 2035 | multiprocessing.cpu_count()), self.toparse) |
| 2015 | 2036 | ||
| 2016 | self.start() | 2037 | self.start() |
| 2017 | self.haveshutdown = False | 2038 | self.haveshutdown = False |
| @@ -2032,7 +2053,7 @@ class CookerParser(object): | |||
| 2032 | 2053 | ||
| 2033 | def chunkify(lst,n): | 2054 | def chunkify(lst,n): |
| 2034 | return [lst[i::n] for i in range(n)] | 2055 | return [lst[i::n] for i in range(n)] |
| 2035 | self.jobs = chunkify(self.willparse, self.num_processes) | 2056 | self.jobs = chunkify(list(self.willparse), self.num_processes) |
| 2036 | 2057 | ||
| 2037 | for i in range(0, self.num_processes): | 2058 | for i in range(0, self.num_processes): |
| 2038 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) | 2059 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) |
| @@ -2095,9 +2116,9 @@ class CookerParser(object): | |||
| 2095 | print("Processed parsing statistics saved to %s" % (pout)) | 2116 | print("Processed parsing statistics saved to %s" % (pout)) |
| 2096 | 2117 | ||
| 2097 | def load_cached(self): | 2118 | def load_cached(self): |
| 2098 | for filename, appends in self.fromcache: | 2119 | for mc, filename, appends in self.fromcache: |
| 2099 | cached, infos = self.bb_cache.load(filename, appends) | 2120 | cached, infos = self.bb_cache.load(mc, filename, appends) |
| 2100 | yield not cached, infos | 2121 | yield not cached, mc, infos |
| 2101 | 2122 | ||
| 2102 | def parse_generator(self): | 2123 | def parse_generator(self): |
| 2103 | while True: | 2124 | while True: |
| @@ -2119,7 +2140,7 @@ class CookerParser(object): | |||
| 2119 | result = [] | 2140 | result = [] |
| 2120 | parsed = None | 2141 | parsed = None |
| 2121 | try: | 2142 | try: |
| 2122 | parsed, result = next(self.results) | 2143 | parsed, mc, result = next(self.results) |
| 2123 | except StopIteration: | 2144 | except StopIteration: |
| 2124 | self.shutdown() | 2145 | self.shutdown() |
| 2125 | return False | 2146 | return False |
| @@ -2181,7 +2202,11 @@ class CookerParser(object): | |||
| 2181 | return True | 2202 | return True |
| 2182 | 2203 | ||
| 2183 | def reparse(self, filename): | 2204 | def reparse(self, filename): |
| 2184 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) | 2205 | to_reparse = set() |
| 2185 | for vfn, info_array in infos: | 2206 | for mc in self.cooker.multiconfigs: |
| 2186 | (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) | 2207 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) |
| 2187 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | 2208 | |
| 2209 | for mc, filename, appends in to_reparse: | ||
| 2210 | infos = self.bb_cache.parse(filename, appends) | ||
| 2211 | for vfn, info_array in infos: | ||
| 2212 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | ||
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 16f076f3b1..3d54c2b88a 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
| @@ -1557,7 +1557,8 @@ class RunQueue: | |||
| 1557 | 1557 | ||
| 1558 | def rq_dump_sigfn(self, fn, options): | 1558 | def rq_dump_sigfn(self, fn, options): |
| 1559 | bb_cache = bb.cache.NoCache(self.cooker.databuilder) | 1559 | bb_cache = bb.cache.NoCache(self.cooker.databuilder) |
| 1560 | the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) | 1560 | mc = bb.runqueue.mc_from_tid(fn) |
| 1561 | the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) | ||
| 1561 | siggen = bb.parse.siggen | 1562 | siggen = bb.parse.siggen |
| 1562 | dataCaches = self.rqdata.dataCaches | 1563 | dataCaches = self.rqdata.dataCaches |
| 1563 | siggen.dump_sigfn(fn, dataCaches, options) | 1564 | siggen.dump_sigfn(fn, dataCaches, options) |
| @@ -2042,10 +2043,10 @@ class RunQueueExecute: | |||
| 2042 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | 2043 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: |
| 2043 | if not mc in self.rq.fakeworker: | 2044 | if not mc in self.rq.fakeworker: |
| 2044 | self.rq.start_fakeworker(self, mc) | 2045 | self.rq.start_fakeworker(self, mc) |
| 2045 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | 2046 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") |
| 2046 | self.rq.fakeworker[mc].process.stdin.flush() | 2047 | self.rq.fakeworker[mc].process.stdin.flush() |
| 2047 | else: | 2048 | else: |
| 2048 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | 2049 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") |
| 2049 | self.rq.worker[mc].process.stdin.flush() | 2050 | self.rq.worker[mc].process.stdin.flush() |
| 2050 | 2051 | ||
| 2051 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | 2052 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) |
| @@ -2129,10 +2130,10 @@ class RunQueueExecute: | |||
| 2129 | self.rq.state = runQueueFailed | 2130 | self.rq.state = runQueueFailed |
| 2130 | self.stats.taskFailed() | 2131 | self.stats.taskFailed() |
| 2131 | return True | 2132 | return True |
| 2132 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") | 2133 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") |
| 2133 | self.rq.fakeworker[mc].process.stdin.flush() | 2134 | self.rq.fakeworker[mc].process.stdin.flush() |
| 2134 | else: | 2135 | else: |
| 2135 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") | 2136 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") |
| 2136 | self.rq.worker[mc].process.stdin.flush() | 2137 | self.rq.worker[mc].process.stdin.flush() |
| 2137 | 2138 | ||
| 2138 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | 2139 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) |
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index 8c9b6b8ca5..dccbe0ebb5 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
| @@ -117,15 +117,16 @@ class TinfoilCookerAdapter: | |||
| 117 | 117 | ||
| 118 | class TinfoilCookerCollectionAdapter: | 118 | class TinfoilCookerCollectionAdapter: |
| 119 | """ cooker.collection adapter """ | 119 | """ cooker.collection adapter """ |
| 120 | def __init__(self, tinfoil): | 120 | def __init__(self, tinfoil, mc=''): |
| 121 | self.tinfoil = tinfoil | 121 | self.tinfoil = tinfoil |
| 122 | self.mc = mc | ||
| 122 | def get_file_appends(self, fn): | 123 | def get_file_appends(self, fn): |
| 123 | return self.tinfoil.get_file_appends(fn) | 124 | return self.tinfoil.get_file_appends(fn, self.mc) |
| 124 | def __getattr__(self, name): | 125 | def __getattr__(self, name): |
| 125 | if name == 'overlayed': | 126 | if name == 'overlayed': |
| 126 | return self.tinfoil.get_overlayed_recipes() | 127 | return self.tinfoil.get_overlayed_recipes(self.mc) |
| 127 | elif name == 'bbappends': | 128 | elif name == 'bbappends': |
| 128 | return self.tinfoil.run_command('getAllAppends') | 129 | return self.tinfoil.run_command('getAllAppends', self.mc) |
| 129 | else: | 130 | else: |
| 130 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) | 131 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) |
| 131 | 132 | ||
| @@ -185,10 +186,11 @@ class TinfoilCookerAdapter: | |||
| 185 | 186 | ||
| 186 | def __init__(self, tinfoil): | 187 | def __init__(self, tinfoil): |
| 187 | self.tinfoil = tinfoil | 188 | self.tinfoil = tinfoil |
| 188 | self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) | 189 | self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() |
| 190 | self.collections = {} | ||
| 189 | self.recipecaches = {} | 191 | self.recipecaches = {} |
| 190 | self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) | 192 | for mc in self.multiconfigs: |
| 191 | for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): | 193 | self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) |
| 192 | self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) | 194 | self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) |
| 193 | self._cache = {} | 195 | self._cache = {} |
| 194 | def __getattr__(self, name): | 196 | def __getattr__(self, name): |
| @@ -492,11 +494,11 @@ class Tinfoil: | |||
| 492 | raise Exception('Not connected to server (did you call .prepare()?)') | 494 | raise Exception('Not connected to server (did you call .prepare()?)') |
| 493 | return self.server_connection.events.waitEvent(timeout) | 495 | return self.server_connection.events.waitEvent(timeout) |
| 494 | 496 | ||
| 495 | def get_overlayed_recipes(self): | 497 | def get_overlayed_recipes(self, mc=''): |
| 496 | """ | 498 | """ |
| 497 | Find recipes which are overlayed (i.e. where recipes exist in multiple layers) | 499 | Find recipes which are overlayed (i.e. where recipes exist in multiple layers) |
| 498 | """ | 500 | """ |
| 499 | return defaultdict(list, self.run_command('getOverlayedRecipes')) | 501 | return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) |
| 500 | 502 | ||
| 501 | def get_skipped_recipes(self): | 503 | def get_skipped_recipes(self): |
| 502 | """ | 504 | """ |
| @@ -534,11 +536,11 @@ class Tinfoil: | |||
| 534 | raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) | 536 | raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) |
| 535 | return best[3] | 537 | return best[3] |
| 536 | 538 | ||
| 537 | def get_file_appends(self, fn): | 539 | def get_file_appends(self, fn, mc=''): |
| 538 | """ | 540 | """ |
| 539 | Find the bbappends for a recipe file | 541 | Find the bbappends for a recipe file |
| 540 | """ | 542 | """ |
| 541 | return self.run_command('getFileAppends', fn) | 543 | return self.run_command('getFileAppends', fn, mc) |
| 542 | 544 | ||
| 543 | def all_recipes(self, mc='', sort=True): | 545 | def all_recipes(self, mc='', sort=True): |
| 544 | """ | 546 | """ |
diff --git a/bitbake/lib/bblayers/action.py b/bitbake/lib/bblayers/action.py index d6459d6617..5b78195ad4 100644 --- a/bitbake/lib/bblayers/action.py +++ b/bitbake/lib/bblayers/action.py | |||
| @@ -143,11 +143,12 @@ build results (as the layer priority order has effectively changed). | |||
| 143 | 143 | ||
| 144 | applied_appends = [] | 144 | applied_appends = [] |
| 145 | for layer in layers: | 145 | for layer in layers: |
| 146 | overlayed = [] | 146 | overlayed = set() |
| 147 | for f in self.tinfoil.cooker.collection.overlayed.keys(): | 147 | for mc in self.tinfoil.cooker.multiconfigs: |
| 148 | for of in self.tinfoil.cooker.collection.overlayed[f]: | 148 | for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): |
| 149 | if of.startswith(layer): | 149 | for of in self.tinfoil.cooker.collections[mc].overlayed[f]: |
| 150 | overlayed.append(of) | 150 | if of.startswith(layer): |
| 151 | overlayed.add(of) | ||
| 151 | 152 | ||
| 152 | logger.plain('Copying files from %s...' % layer ) | 153 | logger.plain('Copying files from %s...' % layer ) |
| 153 | for root, dirs, files in os.walk(layer): | 154 | for root, dirs, files in os.walk(layer): |
| @@ -174,14 +175,21 @@ build results (as the layer priority order has effectively changed). | |||
| 174 | logger.warning('Overwriting file %s', fdest) | 175 | logger.warning('Overwriting file %s', fdest) |
| 175 | bb.utils.copyfile(f1full, fdest) | 176 | bb.utils.copyfile(f1full, fdest) |
| 176 | if ext == '.bb': | 177 | if ext == '.bb': |
| 177 | for append in self.tinfoil.cooker.collection.get_file_appends(f1full): | 178 | appends = set() |
| 179 | for mc in self.tinfoil.cooker.multiconfigs: | ||
| 180 | appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) | ||
| 181 | for append in appends: | ||
| 178 | if layer_path_match(append): | 182 | if layer_path_match(append): |
| 179 | logger.plain(' Applying append %s to %s' % (append, fdest)) | 183 | logger.plain(' Applying append %s to %s' % (append, fdest)) |
| 180 | self.apply_append(append, fdest) | 184 | self.apply_append(append, fdest) |
| 181 | applied_appends.append(append) | 185 | applied_appends.append(append) |
| 182 | 186 | ||
| 183 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes | 187 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes |
| 184 | for b in self.tinfoil.cooker.collection.bbappends: | 188 | bbappends = set() |
| 189 | for mc in self.tinfoil.cooker.multiconfigs: | ||
| 190 | bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) | ||
| 191 | |||
| 192 | for b in bbappends: | ||
| 185 | (recipename, appendname) = b | 193 | (recipename, appendname) = b |
| 186 | if appendname not in applied_appends: | 194 | if appendname not in applied_appends: |
| 187 | first_append = None | 195 | first_append = None |
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py index e2cc310532..ee2db0efed 100644 --- a/bitbake/lib/bblayers/query.py +++ b/bitbake/lib/bblayers/query.py | |||
| @@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. | |||
| 320 | def get_appends_for_files(self, filenames): | 320 | def get_appends_for_files(self, filenames): |
| 321 | appended, notappended = [], [] | 321 | appended, notappended = [], [] |
| 322 | for filename in filenames: | 322 | for filename in filenames: |
| 323 | _, cls, _ = bb.cache.virtualfn2realfn(filename) | 323 | _, cls, mc = bb.cache.virtualfn2realfn(filename) |
| 324 | if cls: | 324 | if cls: |
| 325 | continue | 325 | continue |
| 326 | 326 | ||
| 327 | basename = os.path.basename(filename) | 327 | basename = os.path.basename(filename) |
| 328 | appends = self.tinfoil.cooker.collection.get_file_appends(basename) | 328 | appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) |
| 329 | if appends: | 329 | if appends: |
| 330 | appended.append((basename, list(appends))) | 330 | appended.append((basename, list(appends))) |
| 331 | else: | 331 | else: |
