summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAlejandro Enedino Hernandez Samaniego <alejandro.enedino.hernandez-samaniego@xilinx.com>2018-07-25 09:01:01 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-01 10:07:22 +0100
commitb8730f68d22b9c1ebd012ffd7aedccd22cceb0da (patch)
tree18dcab42041ff11a5c554e1bc798f9df0fc158fc /bitbake/lib
parent7df8a4f834e059e5beddc9f7bedec4f4a5a668d3 (diff)
downloadpoky-b8730f68d22b9c1ebd012ffd7aedccd22cceb0da.tar.gz
bitbake: bitbake: Add support for multiconfig dependencies
This patch adds the capability for tasks from different multiconfigs to depend on one another. These dependencies can be enabled using the following format: task[mcdepends] = "multiconfig:FROM-MC:TO-MC:PN:task-to-depend-on" For the sake of simplicity consider the following example: Assuming we have set up multiconfig builds, one for qemux86 and one for qemuarm, named x86 and arm respectively. Adding the following line to an image recipe (core-image-sato): do_image[mcdepends] = "multiconfig:x86:arm:core-image-minimal:do_rootfs" Would state that core-image-sato:do_image from x86 will depend on core-image-minimal:do_rootfs from arm so it can be executed. This patch makes modifications to: - cooker: To glue both multiconfigs in one place and make sure the dependencies can be provided. - taskdata: To parse and add a new kind of dependency (mcdepends) to the taskdata object. - runqueue: To differentiate tasks from different multiconfigs, add the specified dependencies to the corresponding tasks, and create a working runqueue that contains tasks from both multiconfigs. - siggen: To avoid looking for tasks from different multiconfigs on objects where they dont belong. The taskdata objects are still not aware of the concept of multiconfig, so each object doesnt know which multiconfig its building, hence why the mcdepends are added to all taskdata objects equally (we really dont expect many of these), but the actual dependencies are added only to the required tasks by the runqueue. (Bitbake rev: da8cb8633504bdc815bdcefc538340b9bce5065d) Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/build.py1
-rw-r--r--bitbake/lib/bb/cooker.py34
-rw-r--r--bitbake/lib/bb/runqueue.py21
-rw-r--r--bitbake/lib/bb/siggen.py13
-rw-r--r--bitbake/lib/bb/taskdata.py18
5 files changed, 79 insertions, 8 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 4631abdde5..c79354b3f1 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -803,6 +803,7 @@ def add_tasks(tasklist, d):
803 if name in flags: 803 if name in flags:
804 deptask = d.expand(flags[name]) 804 deptask = d.expand(flags[name])
805 task_deps[name][task] = deptask 805 task_deps[name][task] = deptask
806 getTask('mcdepends')
806 getTask('depends') 807 getTask('depends')
807 getTask('rdepends') 808 getTask('rdepends')
808 getTask('deptask') 809 getTask('deptask')
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 1fda40dd41..946ba9ca06 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -609,7 +609,14 @@ class BBCooker:
609 k2 = k.split(":do_") 609 k2 = k.split(":do_")
610 k = k2[0] 610 k = k2[0]
611 ktask = k2[1] 611 ktask = k2[1]
612 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) 612 if mc:
613 # Provider might be from another mc
614 for mcavailable in self.multiconfigs:
615 # The first element is empty
616 if mcavailable:
617 taskdata[mcavailable].add_provider(localdata[mcavailable], self.recipecaches[mcavailable], k)
618 else:
619 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
613 current += 1 620 current += 1
614 if not ktask.startswith("do_"): 621 if not ktask.startswith("do_"):
615 ktask = "do_%s" % ktask 622 ktask = "do_%s" % ktask
@@ -620,6 +627,27 @@ class BBCooker:
620 runlist.append([mc, k, ktask, fn]) 627 runlist.append([mc, k, ktask, fn])
621 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) 628 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
622 629
630 mcdeps = taskdata[mc].get_mcdepends()
631 # No need to do check providers if there are no mcdeps or not an mc build
632 if mcdeps and mc:
633 # Make sure we can provide the multiconfig dependency
634 seen = set()
635 new = True
636 while new:
637 new = False
638 for mc in self.multiconfigs:
639 for k in mcdeps:
640 if k in seen:
641 continue
642 l = k.split(':')
643 depmc = l[2]
644 if depmc not in self.multiconfigs:
645 bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
646 else:
647 logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
648 taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
649 seen.add(k)
650 new = True
623 for mc in self.multiconfigs: 651 for mc in self.multiconfigs:
624 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) 652 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
625 653
@@ -706,8 +734,8 @@ class BBCooker:
706 if not dotname in depend_tree["tdepends"]: 734 if not dotname in depend_tree["tdepends"]:
707 depend_tree["tdepends"][dotname] = [] 735 depend_tree["tdepends"][dotname] = []
708 for dep in rq.rqdata.runtaskentries[tid].depends: 736 for dep in rq.rqdata.runtaskentries[tid].depends:
709 (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep) 737 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
710 deppn = self.recipecaches[mc].pkg_fn[deptaskfn] 738 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
711 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) 739 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
712 if taskfn not in seen_fns: 740 if taskfn not in seen_fns:
713 seen_fns.append(taskfn) 741 seen_fns.append(taskfn)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index ba9bebebcf..a43c9983a1 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -602,6 +602,19 @@ class RunQueueData:
602 if t in taskData[mc].taskentries: 602 if t in taskData[mc].taskentries:
603 depends.add(t) 603 depends.add(t)
604 604
605 def add_mc_dependencies(mc, tid):
606 mcdeps = taskData[mc].get_mcdepends()
607 for dep in mcdeps:
608 mcdependency = dep.split(':')
609 pn = mcdependency[3]
610 frommc = mcdependency[1]
611 mcdep = mcdependency[2]
612 deptask = mcdependency[4]
613 if mc == frommc:
614 fn = taskData[mcdep].build_targets[pn][0]
615 newdep = '%s:%s' % (fn,deptask)
616 taskData[mc].taskentries[tid].tdepends.append(newdep)
617
605 for mc in taskData: 618 for mc in taskData:
606 for tid in taskData[mc].taskentries: 619 for tid in taskData[mc].taskentries:
607 620
@@ -618,12 +631,16 @@ class RunQueueData:
618 if fn in taskData[mc].failed_fns: 631 if fn in taskData[mc].failed_fns:
619 continue 632 continue
620 633
634 # We add multiconfig dependencies before processing internal task deps (tdepends)
635 if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']:
636 add_mc_dependencies(mc, tid)
637
621 # Resolve task internal dependencies 638 # Resolve task internal dependencies
622 # 639 #
623 # e.g. addtask before X after Y 640 # e.g. addtask before X after Y
624 for t in taskData[mc].taskentries[tid].tdepends: 641 for t in taskData[mc].taskentries[tid].tdepends:
625 (_, depfn, deptaskname, _) = split_tid_mcfn(t) 642 (depmc, depfn, deptaskname, _) = split_tid_mcfn(t)
626 depends.add(build_tid(mc, depfn, deptaskname)) 643 depends.add(build_tid(depmc, depfn, deptaskname))
627 644
628 # Resolve 'deptask' dependencies 645 # Resolve 'deptask' dependencies
629 # 646 #
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index ab228e4148..e9bb51d736 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -193,15 +193,24 @@ class SignatureGeneratorBasic(SignatureGenerator):
193 return taint 193 return taint
194 194
195 def get_taskhash(self, fn, task, deps, dataCache): 195 def get_taskhash(self, fn, task, deps, dataCache):
196
197 mc = ''
198 if fn.startswith('multiconfig:'):
199 mc = fn.split(':')[1]
196 k = fn + "." + task 200 k = fn + "." + task
201
197 data = dataCache.basetaskhash[k] 202 data = dataCache.basetaskhash[k]
198 self.basehash[k] = data 203 self.basehash[k] = data
199 self.runtaskdeps[k] = [] 204 self.runtaskdeps[k] = []
200 self.file_checksum_values[k] = [] 205 self.file_checksum_values[k] = []
201 recipename = dataCache.pkg_fn[fn] 206 recipename = dataCache.pkg_fn[fn]
202
203 for dep in sorted(deps, key=clean_basepath): 207 for dep in sorted(deps, key=clean_basepath):
204 depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] 208 pkgname = self.pkgnameextract.search(dep).group('fn')
209 if mc:
210 depmc = pkgname.split(':')[1]
211 if mc != depmc:
212 continue
213 depname = dataCache.pkg_fn[pkgname]
205 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): 214 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
206 continue 215 continue
207 if dep not in self.taskhash: 216 if dep not in self.taskhash:
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 0ea6c0bfd6..94e822c485 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -70,6 +70,8 @@ class TaskData:
70 70
71 self.skiplist = skiplist 71 self.skiplist = skiplist
72 72
73 self.mcdepends = []
74
73 def add_tasks(self, fn, dataCache): 75 def add_tasks(self, fn, dataCache):
74 """ 76 """
75 Add tasks for a given fn to the database 77 Add tasks for a given fn to the database
@@ -88,6 +90,13 @@ class TaskData:
88 90
89 self.add_extra_deps(fn, dataCache) 91 self.add_extra_deps(fn, dataCache)
90 92
93 def add_mcdepends(task):
94 for dep in task_deps['mcdepends'][task].split():
95 if len(dep.split(':')) != 5:
96 bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
97 if dep not in self.mcdepends:
98 self.mcdepends.append(dep)
99
91 # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends 100 # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
92 def handle_deps(task, dep_name, depends, seen): 101 def handle_deps(task, dep_name, depends, seen):
93 if dep_name in task_deps and task in task_deps[dep_name]: 102 if dep_name in task_deps and task in task_deps[dep_name]:
@@ -110,16 +119,20 @@ class TaskData:
110 parentids = [] 119 parentids = []
111 for dep in task_deps['parents'][task]: 120 for dep in task_deps['parents'][task]:
112 if dep not in task_deps['tasks']: 121 if dep not in task_deps['tasks']:
113 bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep)) 122 bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
114 continue 123 continue
115 parentid = "%s:%s" % (fn, dep) 124 parentid = "%s:%s" % (fn, dep)
116 parentids.append(parentid) 125 parentids.append(parentid)
117 self.taskentries[tid].tdepends.extend(parentids) 126 self.taskentries[tid].tdepends.extend(parentids)
118 127
128
119 # Touch all intertask dependencies 129 # Touch all intertask dependencies
120 handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target) 130 handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
121 handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target) 131 handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
122 132
133 if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
134 add_mcdepends(task)
135
123 # Work out build dependencies 136 # Work out build dependencies
124 if not fn in self.depids: 137 if not fn in self.depids:
125 dependids = set() 138 dependids = set()
@@ -537,6 +550,9 @@ class TaskData:
537 provmap[name] = provider[0] 550 provmap[name] = provider[0]
538 return provmap 551 return provmap
539 552
553 def get_mcdepends(self):
554 return self.mcdepends
555
540 def dump_data(self): 556 def dump_data(self):
541 """ 557 """
542 Dump some debug information on the internal data structures 558 Dump some debug information on the internal data structures