diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-08-16 17:47:06 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-08-18 10:06:27 +0100 |
commit | 218b81acb682bf0006afeb1a5c7bc4adf0549796 (patch) | |
tree | 4048ec43fa894149678209b9f8ae3f3985341c1f /bitbake/lib/bb/cooker.py | |
parent | fac16ff8f7b1090324955e5198996324c6dcdc1d (diff) | |
download | poky-218b81acb682bf0006afeb1a5c7bc4adf0549796.tar.gz |
bitbake: bitbake: Initial multi-config support
This patch adds the notion of supporting multiple configurations within
a single build. To enable it, set a line in local.conf like:
BBMULTICONFIG = "configA configB configC"
This would tell bitbake that before it parses the base configuration,
it should load conf/configA.conf and so on for each different
configuration. These would contain lines like:
MACHINE = "A"
or other variables which can be set which can be built in the same
build directory (or change TMPDIR not to conflict).
One downside I've already discovered is that if we want to inherit this
file right at the start of parsing, the only place you can put the
configurations is in "cwd", since BBPATH isn't constructed until the
layers are parsed and therefore using it as a preconf file isn't
possible unless its located there.
Execution of these targets takes the form "bitbake
multiconfig:configA:core-image-minimal core-image-sato" so similar to
our virtclass approach for native/nativesdk/multilib using BBCLASSEXTEND.
Implementation wise, the implication is that instead of tasks being
uniquely referenced with "recipename/fn:task" it now needs to be
"configuration:recipename:task".
We already started using "virtual" filenames for recipes when we
implemented BBCLASSEXTEND and this patch adds a new prefix to
these, "multiconfig:<configname>:" and hence avoid changes to a large
part of the codebase thanks to this. databuilder has an internal array
of data stores and uses the right one depending on the supplied virtual
filename.
That trick allows us to use the existing parsing code including the
multithreading mostly unchanged as well as most of the cache code.
For recipecache, we end up with a dict of these accessed by
multiconfig (mc). taskdata and runqueue can only cope with one recipecache
so for taskdata, we pass in each recipecache and have it compute the result
and end up with an array of taskdatas. We can only have one runqueue so there
extensive changes there.
This initial implementation has some drawbacks:
a) There are no inter-multi-configuration dependencies as yet
b) There are no sstate optimisations. This means if the build uses the
same object twice in say two different TMPDIRs, it will either load from
an existing sstate cache at the start or build it twice. We can then in
due course look at ways in which it would only build it once and then
reuse it. This will likely need significant changes to the way sstate
currently works to make that possible.
(Bitbake rev: 5287991691578825c847bac2368e9b51c0ede3f0)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 294 |
1 files changed, 169 insertions, 125 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index fe95e73a12..d1ab4aa17b 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -166,7 +166,7 @@ class BBCooker: | |||
166 | """ | 166 | """ |
167 | 167 | ||
168 | def __init__(self, configuration, featureSet=None): | 168 | def __init__(self, configuration, featureSet=None): |
169 | self.recipecache = None | 169 | self.recipecaches = None |
170 | self.skiplist = {} | 170 | self.skiplist = {} |
171 | self.featureset = CookerFeatures() | 171 | self.featureset = CookerFeatures() |
172 | if featureSet: | 172 | if featureSet: |
@@ -521,11 +521,14 @@ class BBCooker: | |||
521 | nice = int(nice) - curnice | 521 | nice = int(nice) - curnice |
522 | buildlog.verbose("Renice to %s " % os.nice(nice)) | 522 | buildlog.verbose("Renice to %s " % os.nice(nice)) |
523 | 523 | ||
524 | if self.recipecache: | 524 | if self.recipecaches: |
525 | del self.recipecache | 525 | del self.recipecaches |
526 | self.recipecache = bb.cache.CacheData(self.caches_array) | 526 | self.multiconfigs = self.databuilder.mcdata.keys() |
527 | self.recipecaches = {} | ||
528 | for mc in self.multiconfigs: | ||
529 | self.recipecaches[mc] = bb.cache.CacheData(self.caches_array) | ||
527 | 530 | ||
528 | self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) ) | 531 | self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS", True)) |
529 | 532 | ||
530 | def updateConfigOpts(self, options, environment): | 533 | def updateConfigOpts(self, options, environment): |
531 | clean = True | 534 | clean = True |
@@ -569,8 +572,8 @@ class BBCooker: | |||
569 | 572 | ||
570 | def showVersions(self): | 573 | def showVersions(self): |
571 | 574 | ||
572 | pkg_pn = self.recipecache.pkg_pn | 575 | pkg_pn = self.recipecaches[''].pkg_pn |
573 | (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecache, pkg_pn) | 576 | (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn) |
574 | 577 | ||
575 | logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") | 578 | logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") |
576 | logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") | 579 | logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") |
@@ -601,17 +604,18 @@ class BBCooker: | |||
601 | # this showEnvironment() code path doesn't use the cache | 604 | # this showEnvironment() code path doesn't use the cache |
602 | self.parseConfiguration() | 605 | self.parseConfiguration() |
603 | 606 | ||
604 | fn, cls = bb.cache.virtualfn2realfn(buildfile) | 607 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
605 | fn = self.matchFile(fn) | 608 | fn = self.matchFile(fn) |
606 | fn = bb.cache.realfn2virtual(fn, cls) | 609 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
607 | elif len(pkgs_to_build) == 1: | 610 | elif len(pkgs_to_build) == 1: |
608 | ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or "" | 611 | ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or "" |
609 | if pkgs_to_build[0] in set(ignore.split()): | 612 | if pkgs_to_build[0] in set(ignore.split()): |
610 | bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) | 613 | bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) |
611 | 614 | ||
612 | taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) | 615 | taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) |
613 | 616 | ||
614 | fn = taskdata.build_targets[pkgs_to_build[0]][0] | 617 | mc = runlist[0][0] |
618 | fn = runlist[0][3] | ||
615 | else: | 619 | else: |
616 | envdata = self.data | 620 | envdata = self.data |
617 | 621 | ||
@@ -652,29 +656,43 @@ class BBCooker: | |||
652 | task = self.configuration.cmd | 656 | task = self.configuration.cmd |
653 | 657 | ||
654 | fulltargetlist = self.checkPackages(pkgs_to_build) | 658 | fulltargetlist = self.checkPackages(pkgs_to_build) |
659 | taskdata = {} | ||
660 | localdata = {} | ||
655 | 661 | ||
656 | localdata = data.createCopy(self.data) | 662 | for mc in self.multiconfigs: |
657 | bb.data.update_data(localdata) | 663 | taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete) |
658 | bb.data.expandKeys(localdata) | 664 | localdata[mc] = data.createCopy(self.databuilder.mcdata[mc]) |
659 | taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete) | 665 | bb.data.update_data(localdata[mc]) |
666 | bb.data.expandKeys(localdata[mc]) | ||
660 | 667 | ||
661 | current = 0 | 668 | current = 0 |
662 | runlist = [] | 669 | runlist = [] |
663 | for k in fulltargetlist: | 670 | for k in fulltargetlist: |
671 | mc = "" | ||
672 | if k.startswith("multiconfig:"): | ||
673 | mc = k.split(":")[1] | ||
674 | k = ":".join(k.split(":")[2:]) | ||
664 | ktask = task | 675 | ktask = task |
665 | if ":do_" in k: | 676 | if ":do_" in k: |
666 | k2 = k.split(":do_") | 677 | k2 = k.split(":do_") |
667 | k = k2[0] | 678 | k = k2[0] |
668 | ktask = k2[1] | 679 | ktask = k2[1] |
669 | taskdata.add_provider(localdata, self.recipecache, k) | 680 | taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) |
670 | current += 1 | 681 | current += 1 |
671 | if not ktask.startswith("do_"): | 682 | if not ktask.startswith("do_"): |
672 | ktask = "do_%s" % ktask | 683 | ktask = "do_%s" % ktask |
673 | runlist.append([k, ktask]) | 684 | if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]: |
685 | # e.g. in ASSUME_PROVIDED | ||
686 | continue | ||
687 | fn = taskdata[mc].build_targets[k][0] | ||
688 | runlist.append([mc, k, ktask, fn]) | ||
674 | bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) | 689 | bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) |
675 | taskdata.add_unresolved(localdata, self.recipecache) | 690 | |
691 | for mc in self.multiconfigs: | ||
692 | taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) | ||
693 | |||
676 | bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data) | 694 | bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data) |
677 | return taskdata, runlist, fulltargetlist | 695 | return taskdata, runlist |
678 | 696 | ||
679 | def prepareTreeData(self, pkgs_to_build, task): | 697 | def prepareTreeData(self, pkgs_to_build, task): |
680 | """ | 698 | """ |
@@ -683,7 +701,7 @@ class BBCooker: | |||
683 | 701 | ||
684 | # We set abort to False here to prevent unbuildable targets raising | 702 | # We set abort to False here to prevent unbuildable targets raising |
685 | # an exception when we're just generating data | 703 | # an exception when we're just generating data |
686 | taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) | 704 | taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) |
687 | 705 | ||
688 | return runlist, taskdata | 706 | return runlist, taskdata |
689 | 707 | ||
@@ -695,10 +713,15 @@ class BBCooker: | |||
695 | information. | 713 | information. |
696 | """ | 714 | """ |
697 | runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) | 715 | runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) |
698 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) | 716 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) |
699 | rq.rqdata.prepare() | 717 | rq.rqdata.prepare() |
700 | return self.buildDependTree(rq, taskdata) | 718 | return self.buildDependTree(rq, taskdata) |
701 | 719 | ||
720 | @staticmethod | ||
721 | def add_mc_prefix(mc, pn): | ||
722 | if mc: | ||
723 | return "multiconfig:%s.%s" % (mc, pn) | ||
724 | return pn | ||
702 | 725 | ||
703 | def buildDependTree(self, rq, taskdata): | 726 | def buildDependTree(self, rq, taskdata): |
704 | seen_fns = [] | 727 | seen_fns = [] |
@@ -711,24 +734,27 @@ class BBCooker: | |||
711 | depend_tree["rdepends-pkg"] = {} | 734 | depend_tree["rdepends-pkg"] = {} |
712 | depend_tree["rrecs-pkg"] = {} | 735 | depend_tree["rrecs-pkg"] = {} |
713 | depend_tree['providermap'] = {} | 736 | depend_tree['providermap'] = {} |
714 | depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities | 737 | depend_tree["layer-priorities"] = self.bbfile_config_priorities |
715 | 738 | ||
716 | for name, fn in list(taskdata.get_providermap().items()): | 739 | for mc in taskdata: |
717 | pn = self.recipecache.pkg_fn[fn] | 740 | for name, fn in list(taskdata[mc].get_providermap().items()): |
718 | if name != pn: | 741 | pn = self.recipecaches[mc].pkg_fn[fn] |
719 | version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] | 742 | pn = self.add_mc_prefix(mc, pn) |
720 | depend_tree['providermap'][name] = (pn, version) | 743 | if name != pn: |
744 | version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn] | ||
745 | depend_tree['providermap'][name] = (pn, version) | ||
721 | 746 | ||
722 | for tid in rq.rqdata.runtaskentries: | 747 | for tid in rq.rqdata.runtaskentries: |
723 | taskname = bb.runqueue.taskname_from_tid(tid) | 748 | (mc, fn, taskname) = bb.runqueue.split_tid(tid) |
724 | fn = bb.runqueue.fn_from_tid(tid) | 749 | taskfn = bb.runqueue.taskfn_fromtid(tid) |
725 | pn = self.recipecache.pkg_fn[fn] | 750 | pn = self.recipecaches[mc].pkg_fn[taskfn] |
726 | version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] | 751 | pn = self.add_mc_prefix(mc, pn) |
752 | version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] | ||
727 | if pn not in depend_tree["pn"]: | 753 | if pn not in depend_tree["pn"]: |
728 | depend_tree["pn"][pn] = {} | 754 | depend_tree["pn"][pn] = {} |
729 | depend_tree["pn"][pn]["filename"] = fn | 755 | depend_tree["pn"][pn]["filename"] = taskfn |
730 | depend_tree["pn"][pn]["version"] = version | 756 | depend_tree["pn"][pn]["version"] = version |
731 | depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None) | 757 | depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) |
732 | 758 | ||
733 | # if we have extra caches, list all attributes they bring in | 759 | # if we have extra caches, list all attributes they bring in |
734 | extra_info = [] | 760 | extra_info = [] |
@@ -739,36 +765,37 @@ class BBCooker: | |||
739 | 765 | ||
740 | # for all attributes stored, add them to the dependency tree | 766 | # for all attributes stored, add them to the dependency tree |
741 | for ei in extra_info: | 767 | for ei in extra_info: |
742 | depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] | 768 | depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] |
743 | 769 | ||
744 | 770 | ||
745 | for dep in rq.rqdata.runtaskentries[tid].depends: | 771 | for dep in rq.rqdata.runtaskentries[tid].depends: |
746 | depfn = bb.runqueue.fn_from_tid(dep) | 772 | (depmc, depfn, deptaskname) = bb.runqueue.split_tid(dep) |
747 | deppn = self.recipecache.pkg_fn[depfn] | 773 | deptaskfn = bb.runqueue.taskfn_fromtid(dep) |
774 | deppn = self.recipecaches[mc].pkg_fn[deptaskfn] | ||
748 | dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid)) | 775 | dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid)) |
749 | if not dotname in depend_tree["tdepends"]: | 776 | if not dotname in depend_tree["tdepends"]: |
750 | depend_tree["tdepends"][dotname] = [] | 777 | depend_tree["tdepends"][dotname] = [] |
751 | depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) | 778 | depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) |
752 | if fn not in seen_fns: | 779 | if taskfn not in seen_fns: |
753 | seen_fns.append(fn) | 780 | seen_fns.append(taskfn) |
754 | packages = [] | 781 | packages = [] |
755 | 782 | ||
756 | depend_tree["depends"][pn] = [] | 783 | depend_tree["depends"][pn] = [] |
757 | for dep in taskdata.depids[fn]: | 784 | for dep in taskdata[mc].depids[taskfn]: |
758 | depend_tree["depends"][pn].append(dep) | 785 | depend_tree["depends"][pn].append(dep) |
759 | 786 | ||
760 | depend_tree["rdepends-pn"][pn] = [] | 787 | depend_tree["rdepends-pn"][pn] = [] |
761 | for rdep in taskdata.rdepids[fn]: | 788 | for rdep in taskdata[mc].rdepids[taskfn]: |
762 | depend_tree["rdepends-pn"][pn].append(rdep) | 789 | depend_tree["rdepends-pn"][pn].append(rdep) |
763 | 790 | ||
764 | rdepends = self.recipecache.rundeps[fn] | 791 | rdepends = self.recipecaches[mc].rundeps[taskfn] |
765 | for package in rdepends: | 792 | for package in rdepends: |
766 | depend_tree["rdepends-pkg"][package] = [] | 793 | depend_tree["rdepends-pkg"][package] = [] |
767 | for rdepend in rdepends[package]: | 794 | for rdepend in rdepends[package]: |
768 | depend_tree["rdepends-pkg"][package].append(rdepend) | 795 | depend_tree["rdepends-pkg"][package].append(rdepend) |
769 | packages.append(package) | 796 | packages.append(package) |
770 | 797 | ||
771 | rrecs = self.recipecache.runrecs[fn] | 798 | rrecs = self.recipecaches[mc].runrecs[taskfn] |
772 | for package in rrecs: | 799 | for package in rrecs: |
773 | depend_tree["rrecs-pkg"][package] = [] | 800 | depend_tree["rrecs-pkg"][package] = [] |
774 | for rdepend in rrecs[package]: | 801 | for rdepend in rrecs[package]: |
@@ -780,7 +807,7 @@ class BBCooker: | |||
780 | if package not in depend_tree["packages"]: | 807 | if package not in depend_tree["packages"]: |
781 | depend_tree["packages"][package] = {} | 808 | depend_tree["packages"][package] = {} |
782 | depend_tree["packages"][package]["pn"] = pn | 809 | depend_tree["packages"][package]["pn"] = pn |
783 | depend_tree["packages"][package]["filename"] = fn | 810 | depend_tree["packages"][package]["filename"] = taskfn |
784 | depend_tree["packages"][package]["version"] = version | 811 | depend_tree["packages"][package]["version"] = version |
785 | 812 | ||
786 | return depend_tree | 813 | return depend_tree |
@@ -807,44 +834,54 @@ class BBCooker: | |||
807 | cachefields = getattr(cache_class, 'cachefields', []) | 834 | cachefields = getattr(cache_class, 'cachefields', []) |
808 | extra_info = extra_info + cachefields | 835 | extra_info = extra_info + cachefields |
809 | 836 | ||
810 | for tid in taskdata.taskentries: | 837 | tids = [] |
811 | fn = bb.runqueue.fn_from_tid(tid) | 838 | for mc in taskdata: |
812 | pn = self.recipecache.pkg_fn[fn] | 839 | for tid in taskdata[mc].taskentries: |
840 | tids.append(tid) | ||
841 | |||
842 | for tid in tids: | ||
843 | (mc, fn, taskname) = bb.runqueue.split_tid(tid) | ||
844 | taskfn = bb.runqueue.taskfn_fromtid(tid) | ||
845 | |||
846 | pn = self.recipecaches[mc].pkg_fn[taskfn] | ||
847 | pn = self.add_mc_prefix(mc, pn) | ||
813 | 848 | ||
814 | if pn not in depend_tree["pn"]: | 849 | if pn not in depend_tree["pn"]: |
815 | depend_tree["pn"][pn] = {} | 850 | depend_tree["pn"][pn] = {} |
816 | depend_tree["pn"][pn]["filename"] = fn | 851 | depend_tree["pn"][pn]["filename"] = taskfn |
817 | version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] | 852 | version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] |
818 | depend_tree["pn"][pn]["version"] = version | 853 | depend_tree["pn"][pn]["version"] = version |
819 | rdepends = self.recipecache.rundeps[fn] | 854 | rdepends = self.recipecaches[mc].rundeps[taskfn] |
820 | rrecs = self.recipecache.runrecs[fn] | 855 | rrecs = self.recipecaches[mc].runrecs[taskfn] |
821 | depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None) | 856 | depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) |
822 | 857 | ||
823 | # for all extra attributes stored, add them to the dependency tree | 858 | # for all extra attributes stored, add them to the dependency tree |
824 | for ei in extra_info: | 859 | for ei in extra_info: |
825 | depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] | 860 | depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] |
826 | 861 | ||
827 | if fn not in seen_fns: | 862 | if taskfn not in seen_fns: |
828 | seen_fns.append(fn) | 863 | seen_fns.append(taskfn) |
829 | 864 | ||
830 | depend_tree["depends"][pn] = [] | 865 | depend_tree["depends"][pn] = [] |
831 | for item in taskdata.depids[fn]: | 866 | for item in taskdata[mc].depids[taskfn]: |
832 | pn_provider = "" | 867 | pn_provider = "" |
833 | if dep in taskdata.build_targets and taskdata.build_targets[dep]: | 868 | if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]: |
834 | fn_provider = taskdata.build_targets[dep][0] | 869 | fn_provider = taskdata[mc].build_targets[dep][0] |
835 | pn_provider = self.recipecache.pkg_fn[fn_provider] | 870 | pn_provider = self.recipecaches[mc].pkg_fn[fn_provider] |
836 | else: | 871 | else: |
837 | pn_provider = item | 872 | pn_provider = item |
873 | pn_provider = self.add_mc_prefix(mc, pn_provider) | ||
838 | depend_tree["depends"][pn].append(pn_provider) | 874 | depend_tree["depends"][pn].append(pn_provider) |
839 | 875 | ||
840 | depend_tree["rdepends-pn"][pn] = [] | 876 | depend_tree["rdepends-pn"][pn] = [] |
841 | for rdep in taskdata.rdepids[fn]: | 877 | for rdep in taskdata[mc].rdepids[taskfn]: |
842 | pn_rprovider = "" | 878 | pn_rprovider = "" |
843 | if rdep in taskdata.run_targets and taskdata.run_targets[rdep]: | 879 | if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]: |
844 | fn_rprovider = taskdata.run_targets[rdep][0] | 880 | fn_rprovider = taskdata[mc].run_targets[rdep][0] |
845 | pn_rprovider = self.recipecache.pkg_fn[fn_rprovider] | 881 | pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider] |
846 | else: | 882 | else: |
847 | pn_rprovider = rdep | 883 | pn_rprovider = rdep |
884 | pn_rprovider = self.add_mc_prefix(mc, pn_rprovider) | ||
848 | depend_tree["rdepends-pn"][pn].append(pn_rprovider) | 885 | depend_tree["rdepends-pn"][pn].append(pn_rprovider) |
849 | 886 | ||
850 | depend_tree["rdepends-pkg"].update(rdepends) | 887 | depend_tree["rdepends-pkg"].update(rdepends) |
@@ -928,7 +965,7 @@ class BBCooker: | |||
928 | # Determine which bbappends haven't been applied | 965 | # Determine which bbappends haven't been applied |
929 | 966 | ||
930 | # First get list of recipes, including skipped | 967 | # First get list of recipes, including skipped |
931 | recipefns = list(self.recipecache.pkg_fn.keys()) | 968 | recipefns = list(self.recipecaches[''].pkg_fn.keys()) |
932 | recipefns.extend(self.skiplist.keys()) | 969 | recipefns.extend(self.skiplist.keys()) |
933 | 970 | ||
934 | # Work out list of bbappends that have been applied | 971 | # Work out list of bbappends that have been applied |
@@ -952,20 +989,21 @@ class BBCooker: | |||
952 | 989 | ||
953 | def handlePrefProviders(self): | 990 | def handlePrefProviders(self): |
954 | 991 | ||
955 | localdata = data.createCopy(self.data) | 992 | for mc in self.multiconfigs: |
956 | bb.data.update_data(localdata) | 993 | localdata = data.createCopy(self.databuilder.mcdata[mc]) |
957 | bb.data.expandKeys(localdata) | 994 | bb.data.update_data(localdata) |
995 | bb.data.expandKeys(localdata) | ||
958 | 996 | ||
959 | # Handle PREFERRED_PROVIDERS | 997 | # Handle PREFERRED_PROVIDERS |
960 | for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split(): | 998 | for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split(): |
961 | try: | 999 | try: |
962 | (providee, provider) = p.split(':') | 1000 | (providee, provider) = p.split(':') |
963 | except: | 1001 | except: |
964 | providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) | 1002 | providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) |
965 | continue | 1003 | continue |
966 | if providee in self.recipecache.preferred and self.recipecache.preferred[providee] != provider: | 1004 | if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider: |
967 | providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecache.preferred[providee]) | 1005 | providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee]) |
968 | self.recipecache.preferred[providee] = provider | 1006 | self.recipecaches[mc].preferred[providee] = provider |
969 | 1007 | ||
970 | def findCoreBaseFiles(self, subdir, configfile): | 1008 | def findCoreBaseFiles(self, subdir, configfile): |
971 | corebase = self.data.getVar('COREBASE', True) or "" | 1009 | corebase = self.data.getVar('COREBASE', True) or "" |
@@ -1060,10 +1098,10 @@ class BBCooker: | |||
1060 | """ | 1098 | """ |
1061 | pkg_list = [] | 1099 | pkg_list = [] |
1062 | 1100 | ||
1063 | for pfn in self.recipecache.pkg_fn: | 1101 | for pfn in self.recipecaches[''].pkg_fn: |
1064 | inherits = self.recipecache.inherits.get(pfn, None) | 1102 | inherits = self.recipecaches[''].inherits.get(pfn, None) |
1065 | if inherits and klass in inherits: | 1103 | if inherits and klass in inherits: |
1066 | pkg_list.append(self.recipecache.pkg_fn[pfn]) | 1104 | pkg_list.append(self.recipecaches[''].pkg_fn[pfn]) |
1067 | 1105 | ||
1068 | return pkg_list | 1106 | return pkg_list |
1069 | 1107 | ||
@@ -1096,10 +1134,10 @@ class BBCooker: | |||
1096 | shell.start( self ) | 1134 | shell.start( self ) |
1097 | 1135 | ||
1098 | 1136 | ||
1099 | def handleCollections( self, collections ): | 1137 | def handleCollections(self, collections): |
1100 | """Handle collections""" | 1138 | """Handle collections""" |
1101 | errors = False | 1139 | errors = False |
1102 | self.recipecache.bbfile_config_priorities = [] | 1140 | self.bbfile_config_priorities = [] |
1103 | if collections: | 1141 | if collections: |
1104 | collection_priorities = {} | 1142 | collection_priorities = {} |
1105 | collection_depends = {} | 1143 | collection_depends = {} |
@@ -1177,7 +1215,7 @@ class BBCooker: | |||
1177 | parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) | 1215 | parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) |
1178 | errors = True | 1216 | errors = True |
1179 | continue | 1217 | continue |
1180 | self.recipecache.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) | 1218 | self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) |
1181 | if errors: | 1219 | if errors: |
1182 | # We've already printed the actual error(s) | 1220 | # We've already printed the actual error(s) |
1183 | raise CollectionError("Errors during parsing layer configuration") | 1221 | raise CollectionError("Errors during parsing layer configuration") |
@@ -1200,7 +1238,7 @@ class BBCooker: | |||
1200 | if bf.startswith("/") or bf.startswith("../"): | 1238 | if bf.startswith("/") or bf.startswith("../"): |
1201 | bf = os.path.abspath(bf) | 1239 | bf = os.path.abspath(bf) |
1202 | 1240 | ||
1203 | self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities) | 1241 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) |
1204 | filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data) | 1242 | filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data) |
1205 | try: | 1243 | try: |
1206 | os.stat(bf) | 1244 | os.stat(bf) |
@@ -1250,7 +1288,7 @@ class BBCooker: | |||
1250 | if (task == None): | 1288 | if (task == None): |
1251 | task = self.configuration.cmd | 1289 | task = self.configuration.cmd |
1252 | 1290 | ||
1253 | fn, cls = bb.cache.virtualfn2realfn(buildfile) | 1291 | fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) |
1254 | fn = self.matchFile(fn) | 1292 | fn = self.matchFile(fn) |
1255 | 1293 | ||
1256 | self.buildSetVars() | 1294 | self.buildSetVars() |
@@ -1260,7 +1298,7 @@ class BBCooker: | |||
1260 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) | 1298 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) |
1261 | infos = dict(infos) | 1299 | infos = dict(infos) |
1262 | 1300 | ||
1263 | fn = bb.cache.realfn2virtual(fn, cls) | 1301 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
1264 | try: | 1302 | try: |
1265 | info_array = infos[fn] | 1303 | info_array = infos[fn] |
1266 | except KeyError: | 1304 | except KeyError: |
@@ -1269,29 +1307,30 @@ class BBCooker: | |||
1269 | if info_array[0].skipped: | 1307 | if info_array[0].skipped: |
1270 | bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason)) | 1308 | bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason)) |
1271 | 1309 | ||
1272 | self.recipecache.add_from_recipeinfo(fn, info_array) | 1310 | self.recipecaches[mc].add_from_recipeinfo(fn, info_array) |
1273 | 1311 | ||
1274 | # Tweak some variables | 1312 | # Tweak some variables |
1275 | item = info_array[0].pn | 1313 | item = info_array[0].pn |
1276 | self.recipecache.ignored_dependencies = set() | 1314 | self.recipecaches[mc].ignored_dependencies = set() |
1277 | self.recipecache.bbfile_priority[fn] = 1 | 1315 | self.recipecaches[mc].bbfile_priority[fn] = 1 |
1278 | 1316 | ||
1279 | # Remove external dependencies | 1317 | # Remove external dependencies |
1280 | self.recipecache.task_deps[fn]['depends'] = {} | 1318 | self.recipecaches[mc].task_deps[fn]['depends'] = {} |
1281 | self.recipecache.deps[fn] = [] | 1319 | self.recipecaches[mc].deps[fn] = [] |
1282 | self.recipecache.rundeps[fn] = [] | 1320 | self.recipecaches[mc].rundeps[fn] = [] |
1283 | self.recipecache.runrecs[fn] = [] | 1321 | self.recipecaches[mc].runrecs[fn] = [] |
1284 | 1322 | ||
1285 | # Invalidate task for target if force mode active | 1323 | # Invalidate task for target if force mode active |
1286 | if self.configuration.force: | 1324 | if self.configuration.force: |
1287 | logger.verbose("Invalidate task %s, %s", task, fn) | 1325 | logger.verbose("Invalidate task %s, %s", task, fn) |
1288 | if not task.startswith("do_"): | 1326 | if not task.startswith("do_"): |
1289 | task = "do_%s" % task | 1327 | task = "do_%s" % task |
1290 | bb.parse.siggen.invalidate_task(task, self.recipecache, fn) | 1328 | bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn) |
1291 | 1329 | ||
1292 | # Setup taskdata structure | 1330 | # Setup taskdata structure |
1293 | taskdata = bb.taskdata.TaskData(self.configuration.abort) | 1331 | taskdata = {} |
1294 | taskdata.add_provider(self.data, self.recipecache, item) | 1332 | taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort) |
1333 | taskdata[mc].add_provider(self.data, self.recipecaches[mc], item) | ||
1295 | 1334 | ||
1296 | buildname = self.data.getVar("BUILDNAME", True) | 1335 | buildname = self.data.getVar("BUILDNAME", True) |
1297 | bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data) | 1336 | bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data) |
@@ -1299,9 +1338,9 @@ class BBCooker: | |||
1299 | # Execute the runqueue | 1338 | # Execute the runqueue |
1300 | if not task.startswith("do_"): | 1339 | if not task.startswith("do_"): |
1301 | task = "do_%s" % task | 1340 | task = "do_%s" % task |
1302 | runlist = [[item, task]] | 1341 | runlist = [[mc, item, task, fn]] |
1303 | 1342 | ||
1304 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) | 1343 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) |
1305 | 1344 | ||
1306 | def buildFileIdle(server, rq, abort): | 1345 | def buildFileIdle(server, rq, abort): |
1307 | 1346 | ||
@@ -1382,23 +1421,20 @@ class BBCooker: | |||
1382 | packages = ["%s:%s" % (target, task) for target in targets] | 1421 | packages = ["%s:%s" % (target, task) for target in targets] |
1383 | bb.event.fire(bb.event.BuildInit(packages), self.expanded_data) | 1422 | bb.event.fire(bb.event.BuildInit(packages), self.expanded_data) |
1384 | 1423 | ||
1385 | taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort) | 1424 | taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort) |
1386 | 1425 | ||
1387 | buildname = self.data.getVar("BUILDNAME", False) | 1426 | buildname = self.data.getVar("BUILDNAME", False) |
1388 | 1427 | ||
1389 | # make targets to always look as <target>:do_<task> | 1428 | # make targets to always look as <target>:do_<task> |
1390 | ntargets = [] | 1429 | ntargets = [] |
1391 | for target in fulltargetlist: | 1430 | for target in runlist: |
1392 | if ":" in target: | 1431 | if target[0]: |
1393 | if ":do_" not in target: | 1432 | ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2])) |
1394 | target = "%s:do_%s" % tuple(target.split(":", 1)) | 1433 | ntargets.append("%s:%s" % (target[1], target[2])) |
1395 | else: | ||
1396 | target = "%s:%s" % (target, task) | ||
1397 | ntargets.append(target) | ||
1398 | 1434 | ||
1399 | bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.data) | 1435 | bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.data) |
1400 | 1436 | ||
1401 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) | 1437 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) |
1402 | if 'universe' in targets: | 1438 | if 'universe' in targets: |
1403 | rq.rqdata.warn_multi_bb = True | 1439 | rq.rqdata.warn_multi_bb = True |
1404 | 1440 | ||
@@ -1513,13 +1549,14 @@ class BBCooker: | |||
1513 | if CookerFeatures.SEND_SANITYEVENTS in self.featureset: | 1549 | if CookerFeatures.SEND_SANITYEVENTS in self.featureset: |
1514 | bb.event.fire(bb.event.SanityCheck(False), self.data) | 1550 | bb.event.fire(bb.event.SanityCheck(False), self.data) |
1515 | 1551 | ||
1516 | ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or "" | 1552 | for mc in self.multiconfigs: |
1517 | self.recipecache.ignored_dependencies = set(ignore.split()) | 1553 | ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED", True) or "" |
1554 | self.recipecaches[mc].ignored_dependencies = set(ignore.split()) | ||
1518 | 1555 | ||
1519 | for dep in self.configuration.extra_assume_provided: | 1556 | for dep in self.configuration.extra_assume_provided: |
1520 | self.recipecache.ignored_dependencies.add(dep) | 1557 | self.recipecaches[mc].ignored_dependencies.add(dep) |
1521 | 1558 | ||
1522 | self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities) | 1559 | self.collection = CookerCollectFiles(self.bbfile_config_priorities) |
1523 | (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data) | 1560 | (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data) |
1524 | 1561 | ||
1525 | self.parser = CookerParser(self, filelist, masked) | 1562 | self.parser = CookerParser(self, filelist, masked) |
@@ -1533,13 +1570,15 @@ class BBCooker: | |||
1533 | raise bb.BBHandledException() | 1570 | raise bb.BBHandledException() |
1534 | self.show_appends_with_no_recipes() | 1571 | self.show_appends_with_no_recipes() |
1535 | self.handlePrefProviders() | 1572 | self.handlePrefProviders() |
1536 | self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data) | 1573 | for mc in self.multiconfigs: |
1574 | self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) | ||
1537 | self.state = state.running | 1575 | self.state = state.running |
1538 | 1576 | ||
1539 | # Send an event listing all stamps reachable after parsing | 1577 | # Send an event listing all stamps reachable after parsing |
1540 | # which the metadata may use to clean up stale data | 1578 | # which the metadata may use to clean up stale data |
1541 | event = bb.event.ReachableStamps(self.recipecache.stamp) | 1579 | for mc in self.multiconfigs: |
1542 | bb.event.fire(event, self.expanded_data) | 1580 | event = bb.event.ReachableStamps(self.recipecaches[mc].stamp) |
1581 | bb.event.fire(event, self.databuilder.mcdata[mc]) | ||
1543 | return None | 1582 | return None |
1544 | 1583 | ||
1545 | return True | 1584 | return True |
@@ -1558,23 +1597,26 @@ class BBCooker: | |||
1558 | parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) | 1597 | parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) |
1559 | 1598 | ||
1560 | if 'world' in pkgs_to_build: | 1599 | if 'world' in pkgs_to_build: |
1561 | bb.providers.buildWorldTargetList(self.recipecache) | ||
1562 | pkgs_to_build.remove('world') | 1600 | pkgs_to_build.remove('world') |
1563 | for t in self.recipecache.world_target: | 1601 | for mc in self.multiconfigs: |
1564 | pkgs_to_build.append(t) | 1602 | bb.providers.buildWorldTargetList(self.recipecaches[mc]) |
1603 | for t in self.recipecaches[mc].world_target: | ||
1604 | if mc: | ||
1605 | t = "multiconfig:" + mc + ":" + t | ||
1606 | pkgs_to_build.append(t) | ||
1565 | 1607 | ||
1566 | if 'universe' in pkgs_to_build: | 1608 | if 'universe' in pkgs_to_build: |
1567 | parselog.warning("The \"universe\" target is only intended for testing and may produce errors.") | 1609 | parselog.warning("The \"universe\" target is only intended for testing and may produce errors.") |
1568 | parselog.debug(1, "collating packages for \"universe\"") | 1610 | parselog.debug(1, "collating packages for \"universe\"") |
1569 | pkgs_to_build.remove('universe') | 1611 | pkgs_to_build.remove('universe') |
1570 | for t in self.recipecache.universe_target: | 1612 | for mc in self.multiconfigs: |
1571 | pkgs_to_build.append(t) | 1613 | for t in self.recipecaches[mc].universe_target: |
1614 | if mc: | ||
1615 | t = "multiconfig:" + mc + ":" + t | ||
1616 | pkgs_to_build.append(t) | ||
1572 | 1617 | ||
1573 | return pkgs_to_build | 1618 | return pkgs_to_build |
1574 | 1619 | ||
1575 | |||
1576 | |||
1577 | |||
1578 | def pre_serve(self): | 1620 | def pre_serve(self): |
1579 | # Empty the environment. The environment will be populated as | 1621 | # Empty the environment. The environment will be populated as |
1580 | # necessary from the data store. | 1622 | # necessary from the data store. |
@@ -1823,7 +1865,7 @@ class CookerCollectFiles(object): | |||
1823 | # Calculate priorities for each file | 1865 | # Calculate priorities for each file |
1824 | matched = set() | 1866 | matched = set() |
1825 | for p in pkgfns: | 1867 | for p in pkgfns: |
1826 | realfn, cls = bb.cache.virtualfn2realfn(p) | 1868 | realfn, cls, mc = bb.cache.virtualfn2realfn(p) |
1827 | priorities[p] = self.calc_bbfile_priority(realfn, matched) | 1869 | priorities[p] = self.calc_bbfile_priority(realfn, matched) |
1828 | 1870 | ||
1829 | # Don't show the warning if the BBFILE_PATTERN did match .bbappend files | 1871 | # Don't show the warning if the BBFILE_PATTERN did match .bbappend files |
@@ -2164,11 +2206,13 @@ class CookerParser(object): | |||
2164 | if info_array[0].skipped: | 2206 | if info_array[0].skipped: |
2165 | self.skipped += 1 | 2207 | self.skipped += 1 |
2166 | self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) | 2208 | self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) |
2167 | self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache, | 2209 | (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn) |
2210 | self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc], | ||
2168 | parsed=parsed, watcher = self.cooker.add_filewatch) | 2211 | parsed=parsed, watcher = self.cooker.add_filewatch) |
2169 | return True | 2212 | return True |
2170 | 2213 | ||
2171 | def reparse(self, filename): | 2214 | def reparse(self, filename): |
2172 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) | 2215 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) |
2173 | for vfn, info_array in infos: | 2216 | for vfn, info_array in infos: |
2174 | self.cooker.recipecache.add_from_recipeinfo(vfn, info_array) | 2217 | (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) |
2218 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | ||