diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-05-09 13:55:01 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-05-25 13:16:24 +0100 |
commit | 695998f921b691f196825e9067b6db399d691e53 (patch) | |
tree | 29e013e9b3c5046d80974618a3c1e0d841fa1d27 | |
parent | 3673a43c22f8adb98945501786077a77a798fb93 (diff) | |
download | poky-695998f921b691f196825e9067b6db399d691e53.tar.gz |
bitbake: cooker: Add FILE_LAYERNAME variable containing the layername for a recipe
There are times when it would be useful for code to know which layer
(or collection in old bitbake terms) it is contained within.
Add support for FILE_LAYERNAME to be set by bitbake when parsing a recipe
so that it is possible to determine this. To do it, we need to pass data
from the cooker into the recipe endpoints, since only the top level cooker
information knows about the layer structure which makes the patch a bit
painful.
The idea is that this would make layer overrides possible:
OVERRIDES .= ":layer-${FILE_LAYERNAME}"
which then opens possibilities like:
WARN_QA:append:layer-core = " patch-fuzz"
as an example where OE-Core could enable specific QA tests only for that
specific layer.
(Bitbake rev: 7090a14b0035842112d073acf7f2ed1a01fdeccf)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-x | bitbake/bin/bitbake-worker | 3 | ||||
-rw-r--r-- | bitbake/lib/bb/cache.py | 4 | ||||
-rw-r--r-- | bitbake/lib/bb/command.py | 5 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 32 | ||||
-rw-r--r-- | bitbake/lib/bb/cookerdata.py | 17 | ||||
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 2 |
6 files changed, 36 insertions, 27 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker index d743ff5105..451e6926bf 100755 --- a/bitbake/bin/bitbake-worker +++ b/bitbake/bin/bitbake-worker | |||
@@ -151,6 +151,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): | |||
151 | taskhash = runtask['taskhash'] | 151 | taskhash = runtask['taskhash'] |
152 | unihash = runtask['unihash'] | 152 | unihash = runtask['unihash'] |
153 | appends = runtask['appends'] | 153 | appends = runtask['appends'] |
154 | layername = runtask['layername'] | ||
154 | taskdepdata = runtask['taskdepdata'] | 155 | taskdepdata = runtask['taskdepdata'] |
155 | quieterrors = runtask['quieterrors'] | 156 | quieterrors = runtask['quieterrors'] |
156 | # We need to setup the environment BEFORE the fork, since | 157 | # We need to setup the environment BEFORE the fork, since |
@@ -262,7 +263,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): | |||
262 | bb.parse.siggen.set_taskhashes(workerdata["newhashes"]) | 263 | bb.parse.siggen.set_taskhashes(workerdata["newhashes"]) |
263 | ret = 0 | 264 | ret = 0 |
264 | 265 | ||
265 | the_data = databuilder.parseRecipe(fn, appends) | 266 | the_data = databuilder.parseRecipe(fn, appends, layername) |
266 | the_data.setVar('BB_TASKHASH', taskhash) | 267 | the_data.setVar('BB_TASKHASH', taskhash) |
267 | the_data.setVar('BB_UNIHASH', unihash) | 268 | the_data.setVar('BB_UNIHASH', unihash) |
268 | bb.parse.siggen.setup_datacache_from_datastore(fn, the_data) | 269 | bb.parse.siggen.setup_datacache_from_datastore(fn, the_data) |
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 10910a6809..5ea41c5de0 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -514,11 +514,11 @@ class Cache(object): | |||
514 | 514 | ||
515 | return len(self.depends_cache) | 515 | return len(self.depends_cache) |
516 | 516 | ||
517 | def parse(self, filename, appends): | 517 | def parse(self, filename, appends, layername): |
518 | """Parse the specified filename, returning the recipe information""" | 518 | """Parse the specified filename, returning the recipe information""" |
519 | self.logger.debug("Parsing %s", filename) | 519 | self.logger.debug("Parsing %s", filename) |
520 | infos = [] | 520 | infos = [] |
521 | datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc) | 521 | datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername) |
522 | depends = [] | 522 | depends = [] |
523 | variants = [] | 523 | variants = [] |
524 | # Process the "real" fn last so we can store variants list | 524 | # Process the "real" fn last so we can store variants list |
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py index 9e2cdc5c75..a355f56c60 100644 --- a/bitbake/lib/bb/command.py +++ b/bitbake/lib/bb/command.py | |||
@@ -561,6 +561,7 @@ class CommandsSync: | |||
561 | appendfiles = command.cooker.collections[mc].get_file_appends(fn) | 561 | appendfiles = command.cooker.collections[mc].get_file_appends(fn) |
562 | else: | 562 | else: |
563 | appendfiles = [] | 563 | appendfiles = [] |
564 | layername = command.cooker.collections[mc].calc_bbfile_priority(fn)[2] | ||
564 | # We are calling bb.cache locally here rather than on the server, | 565 | # We are calling bb.cache locally here rather than on the server, |
565 | # but that's OK because it doesn't actually need anything from | 566 | # but that's OK because it doesn't actually need anything from |
566 | # the server barring the global datastore (which we have a remote | 567 | # the server barring the global datastore (which we have a remote |
@@ -568,10 +569,10 @@ class CommandsSync: | |||
568 | if config_data: | 569 | if config_data: |
569 | # We have to use a different function here if we're passing in a datastore | 570 | # We have to use a different function here if we're passing in a datastore |
570 | # NOTE: we took a copy above, so we don't do it here again | 571 | # NOTE: we took a copy above, so we don't do it here again |
571 | envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc)[''] | 572 | envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)[''] |
572 | else: | 573 | else: |
573 | # Use the standard path | 574 | # Use the standard path |
574 | envdata = command.cooker.databuilder.parseRecipe(fn, appendfiles) | 575 | envdata = command.cooker.databuilder.parseRecipe(fn, appendfiles, layername) |
575 | idx = command.remotedatastores.store(envdata) | 576 | idx = command.remotedatastores.store(envdata) |
576 | return DataStoreConnectionHandle(idx) | 577 | return DataStoreConnectionHandle(idx) |
577 | parseRecipeFile.readonly = True | 578 | parseRecipeFile.readonly = True |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 2a0ef28755..0a21f1c2f8 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -643,7 +643,8 @@ class BBCooker: | |||
643 | 643 | ||
644 | if fn: | 644 | if fn: |
645 | try: | 645 | try: |
646 | envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn)) | 646 | layername = self.collections[mc].calc_bbfile_priority(fn)[2] |
647 | envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn), layername) | ||
647 | except Exception as e: | 648 | except Exception as e: |
648 | parselog.exception("Unable to read %s", fn) | 649 | parselog.exception("Unable to read %s", fn) |
649 | raise | 650 | raise |
@@ -1448,7 +1449,8 @@ class BBCooker: | |||
1448 | 1449 | ||
1449 | bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) | 1450 | bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) |
1450 | 1451 | ||
1451 | infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn)) | 1452 | layername = self.collections[mc].calc_bbfile_priority(fn)[2] |
1453 | infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername) | ||
1452 | infos = dict(infos) | 1454 | infos = dict(infos) |
1453 | 1455 | ||
1454 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 1456 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
@@ -1833,10 +1835,10 @@ class CookerCollectFiles(object): | |||
1833 | self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) | 1835 | self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) |
1834 | 1836 | ||
1835 | def calc_bbfile_priority(self, filename): | 1837 | def calc_bbfile_priority(self, filename): |
1836 | for _, _, regex, pri in self.bbfile_config_priorities: | 1838 | for layername, _, regex, pri in self.bbfile_config_priorities: |
1837 | if regex.match(filename): | 1839 | if regex.match(filename): |
1838 | return pri, regex | 1840 | return pri, regex, layername |
1839 | return 0, None | 1841 | return 0, None, None |
1840 | 1842 | ||
1841 | def get_bbfiles(self): | 1843 | def get_bbfiles(self): |
1842 | """Get list of default .bb files by reading out the current directory""" | 1844 | """Get list of default .bb files by reading out the current directory""" |
@@ -2009,7 +2011,7 @@ class CookerCollectFiles(object): | |||
2009 | # Calculate priorities for each file | 2011 | # Calculate priorities for each file |
2010 | for p in pkgfns: | 2012 | for p in pkgfns: |
2011 | realfn, cls, mc = bb.cache.virtualfn2realfn(p) | 2013 | realfn, cls, mc = bb.cache.virtualfn2realfn(p) |
2012 | priorities[p], regex = self.calc_bbfile_priority(realfn) | 2014 | priorities[p], regex, _ = self.calc_bbfile_priority(realfn) |
2013 | if regex in unmatched_regex: | 2015 | if regex in unmatched_regex: |
2014 | matched_regex.add(regex) | 2016 | matched_regex.add(regex) |
2015 | unmatched_regex.remove(regex) | 2017 | unmatched_regex.remove(regex) |
@@ -2146,7 +2148,7 @@ class Parser(multiprocessing.Process): | |||
2146 | self.results.close() | 2148 | self.results.close() |
2147 | self.results.join_thread() | 2149 | self.results.join_thread() |
2148 | 2150 | ||
2149 | def parse(self, mc, cache, filename, appends): | 2151 | def parse(self, mc, cache, filename, appends, layername): |
2150 | try: | 2152 | try: |
2151 | origfilter = bb.event.LogHandler.filter | 2153 | origfilter = bb.event.LogHandler.filter |
2152 | # Record the filename we're parsing into any events generated | 2154 | # Record the filename we're parsing into any events generated |
@@ -2160,7 +2162,7 @@ class Parser(multiprocessing.Process): | |||
2160 | bb.event.set_class_handlers(self.handlers.copy()) | 2162 | bb.event.set_class_handlers(self.handlers.copy()) |
2161 | bb.event.LogHandler.filter = parse_filter | 2163 | bb.event.LogHandler.filter = parse_filter |
2162 | 2164 | ||
2163 | return True, mc, cache.parse(filename, appends) | 2165 | return True, mc, cache.parse(filename, appends, layername) |
2164 | except Exception as exc: | 2166 | except Exception as exc: |
2165 | tb = sys.exc_info()[2] | 2167 | tb = sys.exc_info()[2] |
2166 | exc.recipe = filename | 2168 | exc.recipe = filename |
@@ -2200,10 +2202,11 @@ class CookerParser(object): | |||
2200 | for mc in self.cooker.multiconfigs: | 2202 | for mc in self.cooker.multiconfigs: |
2201 | for filename in self.mcfilelist[mc]: | 2203 | for filename in self.mcfilelist[mc]: |
2202 | appends = self.cooker.collections[mc].get_file_appends(filename) | 2204 | appends = self.cooker.collections[mc].get_file_appends(filename) |
2205 | layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] | ||
2203 | if not self.bb_caches[mc].cacheValid(filename, appends): | 2206 | if not self.bb_caches[mc].cacheValid(filename, appends): |
2204 | self.willparse.add((mc, self.bb_caches[mc], filename, appends)) | 2207 | self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername)) |
2205 | else: | 2208 | else: |
2206 | self.fromcache.add((mc, self.bb_caches[mc], filename, appends)) | 2209 | self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername)) |
2207 | 2210 | ||
2208 | self.total = len(self.fromcache) + len(self.willparse) | 2211 | self.total = len(self.fromcache) + len(self.willparse) |
2209 | self.toparse = len(self.willparse) | 2212 | self.toparse = len(self.willparse) |
@@ -2314,7 +2317,7 @@ class CookerParser(object): | |||
2314 | self.syncthread.join() | 2317 | self.syncthread.join() |
2315 | 2318 | ||
2316 | def load_cached(self): | 2319 | def load_cached(self): |
2317 | for mc, cache, filename, appends in self.fromcache: | 2320 | for mc, cache, filename, appends, layername in self.fromcache: |
2318 | infos = cache.loadCached(filename, appends) | 2321 | infos = cache.loadCached(filename, appends) |
2319 | yield False, mc, infos | 2322 | yield False, mc, infos |
2320 | 2323 | ||
@@ -2417,9 +2420,10 @@ class CookerParser(object): | |||
2417 | bb.cache.SiggenRecipeInfo.reset() | 2420 | bb.cache.SiggenRecipeInfo.reset() |
2418 | to_reparse = set() | 2421 | to_reparse = set() |
2419 | for mc in self.cooker.multiconfigs: | 2422 | for mc in self.cooker.multiconfigs: |
2420 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) | 2423 | layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] |
2424 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename), layername)) | ||
2421 | 2425 | ||
2422 | for mc, filename, appends in to_reparse: | 2426 | for mc, filename, appends, layername in to_reparse: |
2423 | infos = self.bb_caches[mc].parse(filename, appends) | 2427 | infos = self.bb_caches[mc].parse(filename, appends, layername) |
2424 | for vfn, info_array in infos: | 2428 | for vfn, info_array in infos: |
2425 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | 2429 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) |
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py index adde0e7444..42b8d64685 100644 --- a/bitbake/lib/bb/cookerdata.py +++ b/bitbake/lib/bb/cookerdata.py | |||
@@ -494,8 +494,9 @@ class CookerDataBuilder(object): | |||
494 | return data | 494 | return data |
495 | 495 | ||
496 | @staticmethod | 496 | @staticmethod |
497 | def _parse_recipe(bb_data, bbfile, appends, mc=''): | 497 | def _parse_recipe(bb_data, bbfile, appends, mc, layername): |
498 | bb_data.setVar("__BBMULTICONFIG", mc) | 498 | bb_data.setVar("__BBMULTICONFIG", mc) |
499 | bb_data.setVar("FILE_LAYERNAME", layername) | ||
499 | 500 | ||
500 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | 501 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) |
501 | bb.parse.cached_mtime_noerror(bbfile_loc) | 502 | bb.parse.cached_mtime_noerror(bbfile_loc) |
@@ -505,7 +506,7 @@ class CookerDataBuilder(object): | |||
505 | bb_data = bb.parse.handle(bbfile, bb_data) | 506 | bb_data = bb.parse.handle(bbfile, bb_data) |
506 | return bb_data | 507 | return bb_data |
507 | 508 | ||
508 | def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None): | 509 | def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None): |
509 | """ | 510 | """ |
510 | Load and parse one .bb build file | 511 | Load and parse one .bb build file |
511 | Return the data and whether parsing resulted in the file being skipped | 512 | Return the data and whether parsing resulted in the file being skipped |
@@ -515,32 +516,32 @@ class CookerDataBuilder(object): | |||
515 | (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile) | 516 | (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile) |
516 | bb_data = self.mcdata[mc].createCopy() | 517 | bb_data = self.mcdata[mc].createCopy() |
517 | bb_data.setVar("__ONLYFINALISE", virtual or "default") | 518 | bb_data.setVar("__ONLYFINALISE", virtual or "default") |
518 | datastores = self._parse_recipe(bb_data, bbfile, appends, mc) | 519 | datastores = self._parse_recipe(bb_data, bbfile, appends, mc, layername) |
519 | return datastores | 520 | return datastores |
520 | 521 | ||
521 | if mc is not None: | 522 | if mc is not None: |
522 | bb_data = self.mcdata[mc].createCopy() | 523 | bb_data = self.mcdata[mc].createCopy() |
523 | return self._parse_recipe(bb_data, bbfile, appends, mc) | 524 | return self._parse_recipe(bb_data, bbfile, appends, mc, layername) |
524 | 525 | ||
525 | bb_data = self.data.createCopy() | 526 | bb_data = self.data.createCopy() |
526 | datastores = self._parse_recipe(bb_data, bbfile, appends) | 527 | datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername) |
527 | 528 | ||
528 | for mc in self.mcdata: | 529 | for mc in self.mcdata: |
529 | if not mc: | 530 | if not mc: |
530 | continue | 531 | continue |
531 | bb_data = self.mcdata[mc].createCopy() | 532 | bb_data = self.mcdata[mc].createCopy() |
532 | newstores = self._parse_recipe(bb_data, bbfile, appends, mc) | 533 | newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername) |
533 | for ns in newstores: | 534 | for ns in newstores: |
534 | datastores["mc:%s:%s" % (mc, ns)] = newstores[ns] | 535 | datastores["mc:%s:%s" % (mc, ns)] = newstores[ns] |
535 | 536 | ||
536 | return datastores | 537 | return datastores |
537 | 538 | ||
538 | def parseRecipe(self, virtualfn, appends): | 539 | def parseRecipe(self, virtualfn, appends, layername): |
539 | """ | 540 | """ |
540 | Return a complete set of data for fn. | 541 | Return a complete set of data for fn. |
541 | To do this, we need to parse the file. | 542 | To do this, we need to parse the file. |
542 | """ | 543 | """ |
543 | logger.debug("Parsing %s (full)" % virtualfn) | 544 | logger.debug("Parsing %s (full)" % virtualfn) |
544 | (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn) | 545 | (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn) |
545 | bb_data = self.parseRecipeVariants(virtualfn, appends, virtonly=True) | 546 | bb_data = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername) |
546 | return bb_data[virtual] | 547 | return bb_data[virtual] |
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 02f1474540..1eac2da5e8 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -2166,6 +2166,7 @@ class RunQueueExecute: | |||
2166 | 'unihash' : self.rqdata.get_task_unihash(task), | 2166 | 'unihash' : self.rqdata.get_task_unihash(task), |
2167 | 'quieterrors' : True, | 2167 | 'quieterrors' : True, |
2168 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), | 2168 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), |
2169 | 'layername' : self.cooker.collections[mc].calc_bbfile_priority(taskfn)[2], | ||
2169 | 'taskdepdata' : self.sq_build_taskdepdata(task), | 2170 | 'taskdepdata' : self.sq_build_taskdepdata(task), |
2170 | 'dry_run' : False, | 2171 | 'dry_run' : False, |
2171 | 'taskdep': taskdep, | 2172 | 'taskdep': taskdep, |
@@ -2259,6 +2260,7 @@ class RunQueueExecute: | |||
2259 | 'unihash' : self.rqdata.get_task_unihash(task), | 2260 | 'unihash' : self.rqdata.get_task_unihash(task), |
2260 | 'quieterrors' : False, | 2261 | 'quieterrors' : False, |
2261 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), | 2262 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), |
2263 | 'layername' : self.cooker.collections[mc].calc_bbfile_priority(taskfn)[2], | ||
2262 | 'taskdepdata' : self.build_taskdepdata(task), | 2264 | 'taskdepdata' : self.build_taskdepdata(task), |
2263 | 'dry_run' : self.rqdata.setscene_enforce, | 2265 | 'dry_run' : self.rqdata.setscene_enforce, |
2264 | 'taskdep': taskdep, | 2266 | 'taskdep': taskdep, |