summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-11 17:05:19 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-16 13:53:17 +0100
commit1069c364170ffbf1bff276fd965aeb85efbc22f8 (patch)
tree5ec3000158c05890b3b5bfc0eaf7bfc2a6ece66d /bitbake/lib/bb
parent5333f31fc7555f0a39246d4fcafa71debf71abe2 (diff)
downloadpoky-1069c364170ffbf1bff276fd965aeb85efbc22f8.tar.gz
bitbake: runqueue: Optimise multiconfig with overlapping setscene
Currently if a multiconfig build contains different configurations which have overlapping sstate artefacts, it will build them multiple times. This is clearly suboptimal and not what users want/expect. This adds code to detect this and stall all but one of the setscne tasks so that once its built, it can be found by the other tasks. We take care to iterate the multiconfigs in order so try and avoid dependency loops. We also match on PN+taskname+taskhash since this is what we know sstate in OE-Core would use. There are some tasks even within a multiconfig which match hashes (mostly do_populate_lic tasks) but those have a much higher chance of circular dependency so aren't work attempting to optimise. If a deadlock does occur the build will be slower but there is code to unbreak such a deadlock so it hopefully doens't break anything. Comments are injected into the test tasks so they have different task hashes and a new test for this optimisation is added. (Bitbake rev: a75c5fd6d4ec56836de0be2fe679c81297a080ad) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/runqueue.py60
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass19
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf9
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf1
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf1
-rw-r--r--bitbake/lib/bb/tests/runqueue.py19
6 files changed, 98 insertions, 11 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 00c71070d2..fa848326d8 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -68,6 +68,14 @@ def build_tid(mc, fn, taskname):
68 return "mc:" + mc + ":" + fn + ":" + taskname 68 return "mc:" + mc + ":" + fn + ":" + taskname
69 return fn + ":" + taskname 69 return fn + ":" + taskname
70 70
71# Index used to pair up potentially matching multiconfig tasks
72# We match on PN, taskname and hash being equal
73def pending_hash_index(tid, rqdata):
74 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
75 pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
76 h = rqdata.runtaskentries[tid].hash
77 return pn + ":" + "taskname" + h
78
71class RunQueueStats: 79class RunQueueStats:
72 """ 80 """
73 Holds statistics on the tasks handled by the associated runQueue 81 Holds statistics on the tasks handled by the associated runQueue
@@ -1717,6 +1725,7 @@ class RunQueueExecute:
1717 self.build_stamps = {} 1725 self.build_stamps = {}
1718 self.build_stamps2 = [] 1726 self.build_stamps2 = []
1719 self.failed_tids = [] 1727 self.failed_tids = []
1728 self.sq_deferred = {}
1720 1729
1721 self.stampcache = {} 1730 self.stampcache = {}
1722 1731
@@ -1921,17 +1930,32 @@ class RunQueueExecute:
1921 # Find the next setscene to run 1930 # Find the next setscene to run
1922 for nexttask in self.rqdata.runq_setscene_tids: 1931 for nexttask in self.rqdata.runq_setscene_tids:
1923 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): 1932 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
1924 if nexttask in self.sqdata.unskippable:
1925 logger.debug(2, "Setscene task %s is unskippable" % nexttask)
1926 if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): 1933 if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
1927 if nexttask not in self.rqdata.target_tids: 1934 if nexttask not in self.rqdata.target_tids:
1928 logger.debug(2, "Skipping setscene for task %s" % nexttask) 1935 logger.debug(2, "Skipping setscene for task %s" % nexttask)
1929 self.sq_task_skip(nexttask) 1936 self.sq_task_skip(nexttask)
1930 self.scenequeue_notneeded.add(nexttask) 1937 self.scenequeue_notneeded.add(nexttask)
1938 if nexttask in self.sq_deferred:
1939 del self.sq_deferred[nexttask]
1940 return True
1941 if nexttask in self.sq_deferred:
1942 if self.sq_deferred[nexttask] not in self.runq_complete:
1943 continue
1944 logger.debug(1, "Task %s no longer deferred" % nexttask)
1945 del self.sq_deferred[nexttask]
1946 valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, None, False)
1947 if not valid:
1948 logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
1949 self.sq_task_failoutright(nexttask)
1931 return True 1950 return True
1951 else:
1952 self.sqdata.outrightfail.remove(nexttask)
1932 if nexttask in self.sqdata.outrightfail: 1953 if nexttask in self.sqdata.outrightfail:
1954 logger.debug(2, 'No package found, so skipping setscene task %s', nexttask)
1933 self.sq_task_failoutright(nexttask) 1955 self.sq_task_failoutright(nexttask)
1934 return True 1956 return True
1957 if nexttask in self.sqdata.unskippable:
1958 logger.debug(2, "Setscene task %s is unskippable" % nexttask)
1935 task = nexttask 1959 task = nexttask
1936 break 1960 break
1937 if task is not None: 1961 if task is not None:
@@ -1982,7 +2006,7 @@ class RunQueueExecute:
1982 if self.can_start_task(): 2006 if self.can_start_task():
1983 return True 2007 return True
1984 2008
1985 if not self.sq_live and not self.sqdone: 2009 if not self.sq_live and not self.sqdone and not self.sq_deferred:
1986 logger.info("Setscene tasks completed") 2010 logger.info("Setscene tasks completed")
1987 logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered))) 2011 logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
1988 2012
@@ -2083,6 +2107,13 @@ class RunQueueExecute:
2083 self.rq.read_workers() 2107 self.rq.read_workers()
2084 return self.rq.active_fds() 2108 return self.rq.active_fds()
2085 2109
2110 # No more tasks can be run. If we have deferred setscene tasks we should run them.
2111 if self.sq_deferred:
2112 tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
2113 logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
2114 self.sq_task_failoutright(tid)
2115 return True
2116
2086 if len(self.failed_tids) != 0: 2117 if len(self.failed_tids) != 0:
2087 self.rq.state = runQueueFailed 2118 self.rq.state = runQueueFailed
2088 return True 2119 return True
@@ -2347,7 +2378,7 @@ class SQData(object):
2347 # Setscene tasks directly depended upon by the build 2378 # Setscene tasks directly depended upon by the build
2348 self.unskippable = set() 2379 self.unskippable = set()
2349 # List of setscene tasks which aren't present 2380 # List of setscene tasks which aren't present
2350 self.outrightfail = [] 2381 self.outrightfail = set()
2351 # A list of normal tasks a setscene task covers 2382 # A list of normal tasks a setscene task covers
2352 self.sq_covered_tasks = {} 2383 self.sq_covered_tasks = {}
2353 2384
@@ -2510,7 +2541,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
2510 2541
2511 rqdata.init_progress_reporter.next_stage() 2542 rqdata.init_progress_reporter.next_stage()
2512 2543
2544 multiconfigs = set()
2513 for tid in sqdata.sq_revdeps: 2545 for tid in sqdata.sq_revdeps:
2546 multiconfigs.add(mc_from_tid(tid))
2514 if len(sqdata.sq_revdeps[tid]) == 0: 2547 if len(sqdata.sq_revdeps[tid]) == 0:
2515 sqrq.sq_buildable.add(tid) 2548 sqrq.sq_buildable.add(tid)
2516 2549
@@ -2552,10 +2585,21 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
2552 for v in valid: 2585 for v in valid:
2553 valid_new.append(v) 2586 valid_new.append(v)
2554 2587
2555 for tid in sqdata.sq_revdeps: 2588 hashes = {}
2556 if tid not in valid_new and tid not in noexec: 2589 for mc in sorted(multiconfigs):
2557 logger.debug(2, 'No package found, so skipping setscene task %s', tid) 2590 for tid in sqdata.sq_revdeps:
2558 sqdata.outrightfail.append(tid) 2591 if mc_from_tid(tid) != mc:
2592 continue
2593 if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
2594 sqdata.outrightfail.add(tid)
2595
2596 h = pending_hash_index(tid, rqdata)
2597 if h not in hashes:
2598 hashes[h] = tid
2599 else:
2600 sqrq.sq_deferred[tid] = hashes[h]
2601 bb.warn("Deferring %s after %s" % (tid, hashes[h]))
2602
2559 2603
2560class TaskFailure(Exception): 2604class TaskFailure(Exception):
2561 """ 2605 """
diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
index e174c02dd6..cf38d09224 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -4,7 +4,9 @@ SSTATEVALID ??= ""
4def stamptask(d): 4def stamptask(d):
5 import time 5 import time
6 6
7 thistask = d.expand("${PN}:${BB_CURRENTTASK}") 7 thistask = d.expand("${PN}:${BB_CURRENTTASK}")
8 if d.getVar("BB_CURRENT_MC") != "default":
9 thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
8 if thistask in d.getVar("SLOWTASKS").split(): 10 if thistask in d.getVar("SLOWTASKS").split():
9 bb.note("Slowing task %s" % thistask) 11 bb.note("Slowing task %s" % thistask)
10 time.sleep(0.5) 12 time.sleep(0.5)
@@ -13,48 +15,63 @@ def stamptask(d):
13 f.write(thistask + "\n") 15 f.write(thistask + "\n")
14 16
15python do_fetch() { 17python do_fetch() {
18 # fetch
16 stamptask(d) 19 stamptask(d)
17} 20}
18python do_unpack() { 21python do_unpack() {
22 # unpack
19 stamptask(d) 23 stamptask(d)
20} 24}
21python do_patch() { 25python do_patch() {
26 # patch
22 stamptask(d) 27 stamptask(d)
23} 28}
24python do_populate_lic() { 29python do_populate_lic() {
30 # populate_lic
25 stamptask(d) 31 stamptask(d)
26} 32}
27python do_prepare_recipe_sysroot() { 33python do_prepare_recipe_sysroot() {
34 # prepare_recipe_sysroot
28 stamptask(d) 35 stamptask(d)
29} 36}
30python do_configure() { 37python do_configure() {
38 # configure
31 stamptask(d) 39 stamptask(d)
32} 40}
33python do_compile() { 41python do_compile() {
42 # compile
34 stamptask(d) 43 stamptask(d)
35} 44}
36python do_install() { 45python do_install() {
46 # install
37 stamptask(d) 47 stamptask(d)
38} 48}
39python do_populate_sysroot() { 49python do_populate_sysroot() {
50 # populate_sysroot
40 stamptask(d) 51 stamptask(d)
41} 52}
42python do_package() { 53python do_package() {
54 # package
43 stamptask(d) 55 stamptask(d)
44} 56}
45python do_package_write_ipk() { 57python do_package_write_ipk() {
58 # package_write_ipk
46 stamptask(d) 59 stamptask(d)
47} 60}
48python do_package_write_rpm() { 61python do_package_write_rpm() {
62 # package_write_rpm
49 stamptask(d) 63 stamptask(d)
50} 64}
51python do_packagedata() { 65python do_packagedata() {
66 # packagedata
52 stamptask(d) 67 stamptask(d)
53} 68}
54python do_package_qa() { 69python do_package_qa() {
70 # package_qa
55 stamptask(d) 71 stamptask(d)
56} 72}
57python do_build() { 73python do_build() {
74 # build
58 stamptask(d) 75 stamptask(d)
59} 76}
60do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" 77do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 8c7b754dab..96ee1cd5ec 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -6,6 +6,11 @@ PROVIDES = "${PN}"
6PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" 6PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
7PF = "${BB_CURRENT_MC}:${PN}" 7PF = "${BB_CURRENT_MC}:${PN}"
8export PATH 8export PATH
9STAMP = "${TOPDIR}/stamps/${PN}" 9TMPDIR ??= "${TOPDIR}"
10T = "${TOPDIR}/workdir/${PN}/temp" 10STAMP = "${TMPDIR}/stamps/${PN}"
11T = "${TMPDIR}/workdir/${PN}/temp"
11BB_NUMBER_THREADS = "4" 12BB_NUMBER_THREADS = "4"
13
14BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
15
16include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
new file mode 100644
index 0000000000..ecf23e1c73
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -0,0 +1 @@
TMPDIR = "${TOPDIR}/mc1/"
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
new file mode 100644
index 0000000000..eef338e4cc
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -0,0 +1 @@
TMPDIR = "${TOPDIR}/mc2/"
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index 4a65b5b6e7..f0cea6483f 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -198,3 +198,22 @@ class RunQueueTests(unittest.TestCase):
198 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene'] 198 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
199 self.assertEqual(set(tasks), set(expected)) 199 self.assertEqual(set(tasks), set(expected))
200 200
201 def test_multiconfig_setscene_optimise(self):
202 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
203 extraenv = {
204 "BBMULTICONFIG" : "mc1 mc2",
205 "BB_SIGNATURE_HANDLER" : "basic"
206 }
207 cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
208 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
209 'populate_sysroot_setscene', 'package_qa_setscene']
210 sstatevalid = ""
211 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
212 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
213 ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
214 ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
215 ['mc1:b1:build', 'mc2:b1:build']
216 for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
217 expected.remove(x)
218 self.assertEqual(set(tasks), set(expected))
219