summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/tests
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-12 15:54:54 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-14 17:28:23 +0100
commitabda938d665bacc867e6328926892fbeb414f517 (patch)
tree5623fa1ecef343123679c76838b72032c0e91592 /bitbake/lib/bb/tests
parentc86ae704d312eec4e4a86ef046bc8d5601ef4176 (diff)
downloadpoky-abda938d665bacc867e6328926892fbeb414f517.tar.gz
bitbake: tests/runqueue: Add further hash equivalence tests
Add some extra hash equivalence runqueue tests based on recent scenarios that caused problems during testing. (Bitbake rev: 373b085ead992a725b2230ededd992b4c61a1a05) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/tests')
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass20
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf2
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb1
-rw-r--r--bitbake/lib/bb/tests/runqueue.py155
4 files changed, 166 insertions, 12 deletions
diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
index 138edc3fa9..b57650d591 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -7,7 +7,7 @@ def stamptask(d):
7 thistask = d.expand("${PN}:${BB_CURRENTTASK}") 7 thistask = d.expand("${PN}:${BB_CURRENTTASK}")
8 stampname = d.expand("${TOPDIR}/%s.run" % thistask) 8 stampname = d.expand("${TOPDIR}/%s.run" % thistask)
9 with open(stampname, "a+") as f: 9 with open(stampname, "a+") as f:
10 f.write("\n") 10 f.write(d.getVar("BB_UNIHASH") + "\n")
11 11
12 if d.getVar("BB_CURRENT_MC") != "default": 12 if d.getVar("BB_CURRENT_MC") != "default":
13 thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}") 13 thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
@@ -235,18 +235,28 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
235 235
236 valid = d.getVar("SSTATEVALID").split() 236 valid = d.getVar("SSTATEVALID").split()
237 237
238 for tid in sq_data['hash']: 238 for tid in sorted(sq_data['hash']):
239 n = os.path.basename(bb.runqueue.fn_from_tid(tid)).split(".")[0] + ":do_" + bb.runqueue.taskname_from_tid(tid)[3:] 239 n = os.path.basename(bb.runqueue.fn_from_tid(tid)).split(".")[0] + ":do_" + bb.runqueue.taskname_from_tid(tid)[3:]
240 print(n) 240 print(n)
241 stampfile = d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))
241 if n in valid: 242 if n in valid:
242 bb.note("SState: Found valid sstate for %s" % n) 243 bb.note("SState: Found valid sstate for %s" % n)
243 found.add(tid) 244 found.add(tid)
244 elif os.path.exists(d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))): 245 elif n + ":" + sq_data['hash'][tid] in valid:
245 bb.note("SState: Found valid sstate for %s (already run)" % n) 246 bb.note("SState: Found valid sstate for %s" % n)
246 found.add(tid) 247 found.add(tid)
248 elif os.path.exists(stampfile):
249 with open(stampfile, "r") as f:
250 hash = f.readline().strip()
251 if hash == sq_data['hash'][tid]:
252 bb.note("SState: Found valid sstate for %s (already run)" % n)
253 found.add(tid)
254 else:
255 bb.note("SState: sstate hash didn't match previous run for %s (%s vs %s)" % (n, sq_data['hash'][tid], hash))
256 missed.add(tid)
247 else: 257 else:
248 missed.add(tid) 258 missed.add(tid)
249 bb.note("SState: Found no valid sstate for %s" % n) 259 bb.note("SState: Found no valid sstate for %s (%s)" % (n, sq_data['hash'][tid]))
250 260
251 return found 261 return found
252 262
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 1c61f27607..ab0f6bcfac 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -11,6 +11,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
11T = "${TMPDIR}/workdir/${PN}/temp" 11T = "${TMPDIR}/workdir/${PN}/temp"
12BB_NUMBER_THREADS = "4" 12BB_NUMBER_THREADS = "4"
13 13
14BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE" 14BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID"
15 15
16include conf/multiconfig/${BB_CURRENT_MC}.conf 16include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb b/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb
new file mode 100644
index 0000000000..1588bc8a59
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb
@@ -0,0 +1 @@
DEPENDS = "b1" \ No newline at end of file
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index fbdacccfa1..1103f905f9 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -40,10 +40,12 @@ class RunQueueTests(unittest.TestCase):
40 except subprocess.CalledProcessError as e: 40 except subprocess.CalledProcessError as e:
41 self.fail("Command %s failed with %s" % (cmd, e.output)) 41 self.fail("Command %s failed with %s" % (cmd, e.output))
42 tasks = [] 42 tasks = []
43 with open(builddir + "/task.log", "r") as f: 43 tasklog = builddir + "/task.log"
44 tasks = [line.rstrip() for line in f] 44 if os.path.exists(tasklog):
45 if cleanup: 45 with open(tasklog, "r") as f:
46 os.remove(builddir + "/task.log") 46 tasks = [line.rstrip() for line in f]
47 if cleanup:
48 os.remove(tasklog)
47 return tasks 49 return tasks
48 50
49 def test_no_setscenevalid(self): 51 def test_no_setscenevalid(self):
@@ -229,7 +231,7 @@ class RunQueueTests(unittest.TestCase):
229 self.assertEqual(set(tasks), set(expected)) 231 self.assertEqual(set(tasks), set(expected))
230 232
231 233
232 def test_hashserv(self): 234 def test_hashserv_single(self):
233 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: 235 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
234 extraenv = { 236 extraenv = {
235 "BB_HASHSERVE" : "localhost:0", 237 "BB_HASHSERVE" : "localhost:0",
@@ -248,6 +250,147 @@ class RunQueueTests(unittest.TestCase):
248 self.assertEqual(set(tasks), set(expected)) 250 self.assertEqual(set(tasks), set(expected))
249 cmd = ["bitbake", "a1", "b1"] 251 cmd = ["bitbake", "a1", "b1"]
250 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) 252 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
251 expected = ['a1:' + x for x in setscenetasks] + ['b1:' + x for x in setscenetasks] + ['a1:build', 'b1:build'] 253 expected = ['a1:populate_sysroot', 'a1:package', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
254 'a1:package_write_ipk_setscene', 'a1:package_qa_setscene']
252 self.assertEqual(set(tasks), set(expected)) 255 self.assertEqual(set(tasks), set(expected))
253 256
257 def test_hashserv_double(self):
258 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
259 extraenv = {
260 "BB_HASHSERVE" : "localhost:0",
261 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
262 }
263 cmd = ["bitbake", "a1", "b1", "e1"]
264 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
265 'populate_sysroot_setscene', 'package_qa_setscene']
266 sstatevalid = ""
267 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
268 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks]
269 self.assertEqual(set(tasks), set(expected))
270 cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"]
271 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
272 cmd = ["bitbake", "e1"]
273 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
274 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
275 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
276 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene']
277 self.assertEqual(set(tasks), set(expected))
278
279
280 def test_hashserv_multiple_setscene(self):
281 # Runs e1:do_package_setscene twice
282 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
283 extraenv = {
284 "BB_HASHSERVE" : "localhost:0",
285 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
286 }
287 cmd = ["bitbake", "a1", "b1", "e1"]
288 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
289 'populate_sysroot_setscene', 'package_qa_setscene']
290 sstatevalid = ""
291 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
292 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks]
293 self.assertEqual(set(tasks), set(expected))
294 cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"]
295 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
296 cmd = ["bitbake", "e1"]
297 sstatevalid = "e1:do_package"
298 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="a1:populate_sysroot b1:populate_sysroot")
299 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
300 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
301 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
302 'e1:package_setscene']
303 self.assertEqual(set(tasks), set(expected))
304 for i in expected:
305 if i in ["e1:package_setscene"]:
306 self.assertEqual(tasks.count(i), 4, "%s not in task list four times" % i)
307 else:
308 self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
309
310 def test_hashserv_partial_match(self):
311 # e1:do_package matches initial built but not second hash value
312 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
313 extraenv = {
314 "BB_HASHSERVE" : "localhost:0",
315 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
316 }
317 cmd = ["bitbake", "a1", "b1"]
318 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
319 'populate_sysroot_setscene', 'package_qa_setscene']
320 sstatevalid = ""
321 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
322 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
323 self.assertEqual(set(tasks), set(expected))
324 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
325 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
326 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
327 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
328 cmd = ["bitbake", "e1"]
329 sstatevalid = "e1:do_package:cb47e017ab549d87aab614c0f49dcf969ff6414745909094f0af7e657cedc657"
330 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
331 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
332 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
333 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
334 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
335 expected.remove('e1:package')
336 self.assertEqual(set(tasks), set(expected))
337
338 def test_hashserv_partial_match2(self):
339 # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value
340 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
341 extraenv = {
342 "BB_HASHSERVE" : "localhost:0",
343 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
344 }
345 cmd = ["bitbake", "a1", "b1"]
346 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
347 'populate_sysroot_setscene', 'package_qa_setscene']
348 sstatevalid = ""
349 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
350 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
351 self.assertEqual(set(tasks), set(expected))
352 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
353 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
354 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
355 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
356 cmd = ["bitbake", "e1"]
357 sstatevalid = "e1:do_package:cb47e017ab549d87aab614c0f49dcf969ff6414745909094f0af7e657cedc657 e1:do_populate_sysroot:aa6a915229f04af429d3c6c59c303516c500650b7c48da8e07b20a53acd86c5f"
358 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
359 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
360 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
361 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
362 'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata']
363 self.assertEqual(set(tasks), set(expected))
364
365
366 def test_hashserv_partial_match3(self):
367 # e1:do_package is valid for a1 but not after b1
368 # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run
369 # with none of the intermediate tasks which is a serious bug
370 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
371 extraenv = {
372 "BB_HASHSERVE" : "localhost:0",
373 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
374 }
375 cmd = ["bitbake", "a1", "b1"]
376 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
377 'populate_sysroot_setscene', 'package_qa_setscene']
378 sstatevalid = ""
379 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
380 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
381 self.assertEqual(set(tasks), set(expected))
382 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
383 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
384 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
385 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
386 cmd = ["bitbake", "e1", "-DD"]
387 sstatevalid = "e1:do_package:b710f6312ffed900b4b2761cc05538645f4ff3e7e0b70d688c70c0f3bcc2e1a2"
388 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch")
389 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
390 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
391 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
392 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
393 expected.remove('e1:package')
394 self.assertEqual(set(tasks), set(expected))
395
396