summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-02 15:32:27 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-06 11:21:32 +0100
commitfd79638046891fb5ab6091f7ef3e7ae6df12c39b (patch)
tree04a256b8a35158fbb1d90b6d4fbcb84cb2301066 /bitbake/lib/bb
parent347c2056022d21deea6e2e7359eca2b96c6f01fe (diff)
downloadpoky-fd79638046891fb5ab6091f7ef3e7ae6df12c39b.tar.gz
bitbake: tests/runqueue: Add hashserv+runqueue test
Add a test which tests the runqueue adaptations for hash equivalency. (Bitbake rev: 477321d0780df177c1582db119c2bb6795912fc6) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/siggen.py23
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass14
-rw-r--r--bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf2
-rw-r--r--bitbake/lib/bb/tests/runqueue.py27
4 files changed, 59 insertions, 7 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 912c92c8be..b503559305 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -12,6 +12,7 @@ import bb.data
12import difflib 12import difflib
13import simplediff 13import simplediff
14from bb.checksum import FileChecksumCache 14from bb.checksum import FileChecksumCache
15from bb import runqueue
15 16
16logger = logging.getLogger('BitBake.SigGen') 17logger = logging.getLogger('BitBake.SigGen')
17 18
@@ -473,10 +474,12 @@ class SignatureGeneratorUniHashMixIn(object):
473 474
474 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} 475 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
475 476
476 (module, method) = self.method.rsplit('.', 1) 477 if "." in self.method:
477 locs['method'] = getattr(importlib.import_module(module), method) 478 (module, method) = self.method.rsplit('.', 1)
478 479 locs['method'] = getattr(importlib.import_module(module), method)
479 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) 480 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
481 else:
482 outhash = bb.utils.better_eval(self.method + '(path, sigfile, task, d)', locs)
480 483
481 try: 484 try:
482 url = '%s/v1/equivalent' % self.server 485 url = '%s/v1/equivalent' % self.server
@@ -527,6 +530,18 @@ class SignatureGeneratorUniHashMixIn(object):
527 except OSError: 530 except OSError:
528 pass 531 pass
529 532
533
534#
535# Dummy class used for bitbake-selftest
536#
537class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureGeneratorBasicHash):
538 name = "TestEquivHash"
539 def init_rundepcheck(self, data):
540 super().init_rundepcheck(data)
541 self.server = "http://" + data.getVar('BB_HASHSERVE')
542 self.method = "sstate_output_hash"
543
544
530def dump_this_task(outfile, d): 545def dump_this_task(outfile, d):
531 import bb.parse 546 import bb.parse
532 fn = d.getVar("BB_FILENAME") 547 fn = d.getVar("BB_FILENAME")
diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
index 3a0f151c9a..138edc3fa9 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -5,7 +5,8 @@ def stamptask(d):
5 import time 5 import time
6 6
7 thistask = d.expand("${PN}:${BB_CURRENTTASK}") 7 thistask = d.expand("${PN}:${BB_CURRENTTASK}")
8 with open(d.expand("${TOPDIR}/%s.run") % thistask, "a+") as f: 8 stampname = d.expand("${TOPDIR}/%s.run" % thistask)
9 with open(stampname, "a+") as f:
9 f.write("\n") 10 f.write("\n")
10 11
11 if d.getVar("BB_CURRENT_MC") != "default": 12 if d.getVar("BB_CURRENT_MC") != "default":
@@ -13,10 +14,21 @@ def stamptask(d):
13 if thistask in d.getVar("SLOWTASKS").split(): 14 if thistask in d.getVar("SLOWTASKS").split():
14 bb.note("Slowing task %s" % thistask) 15 bb.note("Slowing task %s" % thistask)
15 time.sleep(0.5) 16 time.sleep(0.5)
17 if d.getVar("BB_HASHSERVE"):
18 task = d.getVar("BB_CURRENTTASK")
19 if task in ['package', 'package_qa', 'packagedata', 'package_write_ipk', 'package_write_rpm', 'populate_lic', 'populate_sysroot']:
20 bb.parse.siggen.report_unihash(os.getcwd(), d.getVar("BB_CURRENTTASK"), d)
16 21
17 with open(d.expand("${TOPDIR}/task.log"), "a+") as f: 22 with open(d.expand("${TOPDIR}/task.log"), "a+") as f:
18 f.write(thistask + "\n") 23 f.write(thistask + "\n")
19 24
25
26def sstate_output_hash(path, sigfile, task, d):
27 import hashlib
28 h = hashlib.sha256()
29 h.update(d.expand("${PN}:${BB_CURRENTTASK}").encode('utf-8'))
30 return h.hexdigest()
31
20python do_fetch() { 32python do_fetch() {
21 # fetch 33 # fetch
22 stamptask(d) 34 stamptask(d)
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 96ee1cd5ec..1c61f27607 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -11,6 +11,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
11T = "${TMPDIR}/workdir/${PN}/temp" 11T = "${TMPDIR}/workdir/${PN}/temp"
12BB_NUMBER_THREADS = "4" 12BB_NUMBER_THREADS = "4"
13 13
14BB_HASHBASE_WHITELIST = "BB_CURRENT_MC" 14BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE"
15 15
16include conf/multiconfig/${BB_CURRENT_MC}.conf 16include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index f22ad4bd81..fbdacccfa1 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -25,7 +25,7 @@ class RunQueueTests(unittest.TestCase):
25 a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot" 25 a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
26 b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot" 26 b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
27 27
28 def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None): 28 def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
29 env = os.environ.copy() 29 env = os.environ.copy()
30 env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests")) 30 env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
31 env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS" 31 env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
@@ -42,6 +42,8 @@ class RunQueueTests(unittest.TestCase):
42 tasks = [] 42 tasks = []
43 with open(builddir + "/task.log", "r") as f: 43 with open(builddir + "/task.log", "r") as f:
44 tasks = [line.rstrip() for line in f] 44 tasks = [line.rstrip() for line in f]
45 if cleanup:
46 os.remove(builddir + "/task.log")
45 return tasks 47 return tasks
46 48
47 def test_no_setscenevalid(self): 49 def test_no_setscenevalid(self):
@@ -226,3 +228,26 @@ class RunQueueTests(unittest.TestCase):
226 expected.remove(x) 228 expected.remove(x)
227 self.assertEqual(set(tasks), set(expected)) 229 self.assertEqual(set(tasks), set(expected))
228 230
231
232 def test_hashserv(self):
233 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
234 extraenv = {
235 "BB_HASHSERVE" : "localhost:0",
236 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
237 }
238 cmd = ["bitbake", "a1", "b1"]
239 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
240 'populate_sysroot_setscene', 'package_qa_setscene']
241 sstatevalid = ""
242 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
243 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
244 self.assertEqual(set(tasks), set(expected))
245 cmd = ["bitbake", "a1", "-c", "install", "-f"]
246 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
247 expected = ['a1:install']
248 self.assertEqual(set(tasks), set(expected))
249 cmd = ["bitbake", "a1", "b1"]
250 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
251 expected = ['a1:' + x for x in setscenetasks] + ['b1:' + x for x in setscenetasks] + ['a1:build', 'b1:build']
252 self.assertEqual(set(tasks), set(expected))
253