summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@kernel.crashing.org>2020-09-02 10:33:22 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-09-03 09:44:30 +0100
commit39c1accf9054e863d543df7584eb48e2b0797cea (patch)
treef4fbcc03bc09ac7f687030dce21720c095693e04
parent9189aaedc76b798b9ddc76d28b97f05149300a6b (diff)
downloadpoky-39c1accf9054e863d543df7584eb48e2b0797cea.tar.gz
package.bbclass: hash equivalency and pr service
When the PR service is enabled a number of small changes may happen to variables. In the do_package step a call to package_get_auto_pr will end up setting PRAUTO and modifying PKGV (if AUTOINC is there). PRAUTO is then used by EXTENDPRAUTO, which is then used to generate PKGR. Since this behavior typically happens BEFORE the BB_UNIHASH is calculated for do_package, we need a way to defer the expansion until after we have the unihash value. Writing out the pkgdata files w/o AUTOPR and PKGV (AUTOINC) expanded to placeholder values is the easiest way to deal with this. All other variables are expanded as expected. In the next task, typically do_packagedata, we will then use the UNIHASH from the do_package to get the PR (AUTOPR) as well as generate the AUTOINC replacement value (now PRSERV_PV_AUTOINC). The do_packagedata then translates the placeholders to the final values when copying the data from pkgdata to pkgdata-pdata-input. Also update the prservice test case. With unihash, just changing the do_package (via a _append) will not change the PR. So write the date to a specific file that is incorporated into the unihash to ensure it is always different for the test. Various assert messages were also updated to make it easier to figure out where/why a problem occured. (From OE-Core rev: 2e32f37b0e4abc438c8f60e673cd18a5cc110768) Signed-off-by: Mark Hatle <mark.hatle@kernel.crashing.org> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/package.bbclass58
-rw-r--r--meta/conf/bitbake.conf1
-rw-r--r--meta/lib/oeqa/selftest/cases/prservice.py8
3 files changed, 55 insertions, 12 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 464ba8dc6f..e6236c0bb2 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -7,7 +7,7 @@
7# 7#
8# There are the following default steps but PACKAGEFUNCS can be extended: 8# There are the following default steps but PACKAGEFUNCS can be extended:
9# 9#
10# a) package_get_auto_pr - get PRAUTO from remote PR service 10# a) package_convert_pr_autoinc - convert AUTOINC in PKGV to ${PRSERV_PV_AUTOINC}
11# 11#
12# b) perform_packagecopy - Copy D into PKGD 12# b) perform_packagecopy - Copy D into PKGD
13# 13#
@@ -664,12 +664,20 @@ def runtime_mapping_rename (varname, pkg, d):
664 #bb.note("%s after: %s" % (varname, d.getVar(varname))) 664 #bb.note("%s after: %s" % (varname, d.getVar(varname)))
665 665
666# 666#
667# Package functions suitable for inclusion in PACKAGEFUNCS 667# Used by do_packagedata (and possibly other routines post do_package)
668# 668#
669 669
670package_get_auto_pr[vardepsexclude] = "BB_TASKDEPDATA"
670python package_get_auto_pr() { 671python package_get_auto_pr() {
671 import oe.prservice 672 import oe.prservice
672 import re 673
674 def get_do_package_hash(pn):
675 if d.getVar("BB_RUNTASK") != "do_package":
676 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
677 for dep in taskdepdata:
678 if taskdepdata[dep][1] == "do_package" and taskdepdata[dep][0] == pn:
679 return taskdepdata[dep][6]
680 return None
673 681
674 # Support per recipe PRSERV_HOST 682 # Support per recipe PRSERV_HOST
675 pn = d.getVar('PN') 683 pn = d.getVar('PN')
@@ -681,15 +689,22 @@ python package_get_auto_pr() {
681 689
682 # PR Server not active, handle AUTOINC 690 # PR Server not active, handle AUTOINC
683 if not d.getVar('PRSERV_HOST'): 691 if not d.getVar('PRSERV_HOST'):
684 if 'AUTOINC' in pkgv: 692 d.setVar("PRSERV_PV_AUTOINC", "0")
685 d.setVar("PKGV", pkgv.replace("AUTOINC", "0"))
686 return 693 return
687 694
688 auto_pr = None 695 auto_pr = None
689 pv = d.getVar("PV") 696 pv = d.getVar("PV")
690 version = d.getVar("PRAUTOINX") 697 version = d.getVar("PRAUTOINX")
691 pkgarch = d.getVar("PACKAGE_ARCH") 698 pkgarch = d.getVar("PACKAGE_ARCH")
692 checksum = d.getVar("BB_TASKHASH") 699 checksum = get_do_package_hash(pn)
700
701 # If do_package isn't in the dependencies, we can't get the checksum...
702 if not checksum:
703 bb.warn('Task %s requested do_package unihash, but it was not available.' % d.getVar('BB_RUNTASK'))
704 #taskdepdata = d.getVar("BB_TASKDEPDATA", False)
705 #for dep in taskdepdata:
706 # bb.warn('%s:%s = %s' % (taskdepdata[dep][0], taskdepdata[dep][1], taskdepdata[dep][6]))
707 return
693 708
694 if d.getVar('PRSERV_LOCKDOWN'): 709 if d.getVar('PRSERV_LOCKDOWN'):
695 auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch) or d.getVar('PRAUTO_' + version) or None 710 auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch) or d.getVar('PRAUTO_' + version) or None
@@ -707,7 +722,7 @@ python package_get_auto_pr() {
707 srcpv = bb.fetch2.get_srcrev(d) 722 srcpv = bb.fetch2.get_srcrev(d)
708 base_ver = "AUTOINC-%s" % version[:version.find(srcpv)] 723 base_ver = "AUTOINC-%s" % version[:version.find(srcpv)]
709 value = conn.getPR(base_ver, pkgarch, srcpv) 724 value = conn.getPR(base_ver, pkgarch, srcpv)
710 d.setVar("PKGV", pkgv.replace("AUTOINC", str(value))) 725 d.setVar("PRSERV_PV_AUTOINC", str(value))
711 726
712 auto_pr = conn.getPR(version, pkgarch, checksum) 727 auto_pr = conn.getPR(version, pkgarch, checksum)
713 except Exception as e: 728 except Exception as e:
@@ -717,6 +732,22 @@ python package_get_auto_pr() {
717 d.setVar('PRAUTO',str(auto_pr)) 732 d.setVar('PRAUTO',str(auto_pr))
718} 733}
719 734
735#
736# Package functions suitable for inclusion in PACKAGEFUNCS
737#
738
739python package_convert_pr_autoinc() {
740 pkgv = d.getVar("PKGV")
741
742 # Adjust pkgv as necessary...
743 if 'AUTOINC' in pkgv:
744 d.setVar("PKGV", pkgv.replace("AUTOINC", "${PRSERV_PV_AUTOINC}"))
745
746 # Change PRSERV_PV_AUTOINC and EXTENDPRAUTO usage to special values
747 d.setVar('PRSERV_PV_AUTOINC', '@PRSERV_PV_AUTOINC@')
748 d.setVar('EXTENDPRAUTO', '@EXTENDPRAUTO@')
749}
750
720LOCALEBASEPN ??= "${PN}" 751LOCALEBASEPN ??= "${PN}"
721 752
722python package_do_split_locales() { 753python package_do_split_locales() {
@@ -2335,7 +2366,7 @@ python do_package () {
2335 package_qa_handle_error("var-undefined", msg, d) 2366 package_qa_handle_error("var-undefined", msg, d)
2336 return 2367 return
2337 2368
2338 bb.build.exec_func("package_get_auto_pr", d) 2369 bb.build.exec_func("package_convert_pr_autoinc", d)
2339 2370
2340 ########################################################################### 2371 ###########################################################################
2341 # Optimisations 2372 # Optimisations
@@ -2407,9 +2438,20 @@ addtask do_package_setscene
2407# Copy from PKGDESTWORK to tempdirectory as tempdirectory can be cleaned at both 2438# Copy from PKGDESTWORK to tempdirectory as tempdirectory can be cleaned at both
2408# do_package_setscene and do_packagedata_setscene leading to races 2439# do_package_setscene and do_packagedata_setscene leading to races
2409python do_packagedata () { 2440python do_packagedata () {
2441 bb.build.exec_func("package_get_auto_pr", d)
2442
2410 src = d.expand("${PKGDESTWORK}") 2443 src = d.expand("${PKGDESTWORK}")
2411 dest = d.expand("${WORKDIR}/pkgdata-pdata-input") 2444 dest = d.expand("${WORKDIR}/pkgdata-pdata-input")
2412 oe.path.copyhardlinktree(src, dest) 2445 oe.path.copyhardlinktree(src, dest)
2446
2447 bb.build.exec_func("packagedata_translate_pr_autoinc", d)
2448}
2449
2450# Translate the EXTENDPRAUTO and AUTOINC to the final values
2451packagedata_translate_pr_autoinc() {
2452 find ${WORKDIR}/pkgdata-pdata-input -type f | xargs --no-run-if-empty \
2453 sed -e 's,@PRSERV_PV_AUTOINC@,${PRSERV_PV_AUTOINC},g' \
2454 -e 's,@EXTENDPRAUTO@,${EXTENDPRAUTO},g' -i
2413} 2455}
2414 2456
2415addtask packagedata before do_build after do_package 2457addtask packagedata before do_build after do_package
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 353caacef9..65b4432c63 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -208,6 +208,7 @@ PF = "${PN}-${EXTENDPE}${PV}-${PR}"
208EXTENDPE = "${@['','${PE}_'][int(d.getVar('PE') or 0) > 0]}" 208EXTENDPE = "${@['','${PE}_'][int(d.getVar('PE') or 0) > 0]}"
209P = "${PN}-${PV}" 209P = "${PN}-${PV}"
210 210
211PRSERV_PV_AUTOINC = "AUTOINC"
211PRAUTO = "" 212PRAUTO = ""
212EXTENDPRAUTO = "${@['.${PRAUTO}', ''][not d.getVar('PRAUTO')]}" 213EXTENDPRAUTO = "${@['.${PRAUTO}', ''][not d.getVar('PRAUTO')]}"
213PRAUTOINX = "${PF}" 214PRAUTOINX = "${PF}"
diff --git a/meta/lib/oeqa/selftest/cases/prservice.py b/meta/lib/oeqa/selftest/cases/prservice.py
index 85b534963d..578b2b4dd9 100644
--- a/meta/lib/oeqa/selftest/cases/prservice.py
+++ b/meta/lib/oeqa/selftest/cases/prservice.py
@@ -23,7 +23,7 @@ class BitbakePrTests(OESelftestTestCase):
23 package_data_file = os.path.join(self.pkgdata_dir, 'runtime', package_name) 23 package_data_file = os.path.join(self.pkgdata_dir, 'runtime', package_name)
24 package_data = ftools.read_file(package_data_file) 24 package_data = ftools.read_file(package_data_file)
25 find_pr = re.search(r"PKGR: r[0-9]+\.([0-9]+)", package_data) 25 find_pr = re.search(r"PKGR: r[0-9]+\.([0-9]+)", package_data)
26 self.assertTrue(find_pr, "No PKG revision found in %s" % package_data_file) 26 self.assertTrue(find_pr, "No PKG revision found via regex 'PKGR: r[0-9]+\.([0-9]+)' in %s" % package_data_file)
27 return int(find_pr.group(1)) 27 return int(find_pr.group(1))
28 28
29 def get_task_stamp(self, package_name, recipe_task): 29 def get_task_stamp(self, package_name, recipe_task):
@@ -40,7 +40,7 @@ class BitbakePrTests(OESelftestTestCase):
40 return str(stamps[0]) 40 return str(stamps[0])
41 41
42 def increment_package_pr(self, package_name): 42 def increment_package_pr(self, package_name):
43 inc_data = "do_package_append() {\n bb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() 43 inc_data = "do_package_append() {\n bb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\" > ${PKGDESTWORK}/${PN}.datestamp\n}" % datetime.datetime.now()
44 self.write_recipeinc(package_name, inc_data) 44 self.write_recipeinc(package_name, inc_data)
45 res = bitbake(package_name, ignore_status=True) 45 res = bitbake(package_name, ignore_status=True)
46 self.delete_recipeinc(package_name) 46 self.delete_recipeinc(package_name)
@@ -63,7 +63,7 @@ class BitbakePrTests(OESelftestTestCase):
63 pr_2 = self.get_pr_version(package_name) 63 pr_2 = self.get_pr_version(package_name)
64 stamp_2 = self.get_task_stamp(package_name, track_task) 64 stamp_2 = self.get_task_stamp(package_name, track_task)
65 65
66 self.assertTrue(pr_2 - pr_1 == 1, "Step between pkg revisions is not 1 (was %s - %s)" % (pr_2, pr_1)) 66 self.assertTrue(pr_2 - pr_1 == 1, "New PR %s did not increment as expected (from %s), difference should be 1" % (pr_2, pr_1))
67 self.assertTrue(stamp_1 != stamp_2, "Different pkg rev. but same stamp: %s" % stamp_1) 67 self.assertTrue(stamp_1 != stamp_2, "Different pkg rev. but same stamp: %s" % stamp_1)
68 68
69 def run_test_pr_export_import(self, package_name, replace_current_db=True): 69 def run_test_pr_export_import(self, package_name, replace_current_db=True):
@@ -89,7 +89,7 @@ class BitbakePrTests(OESelftestTestCase):
89 self.increment_package_pr(package_name) 89 self.increment_package_pr(package_name)
90 pr_2 = self.get_pr_version(package_name) 90 pr_2 = self.get_pr_version(package_name)
91 91
92 self.assertTrue(pr_2 - pr_1 == 1, "Step between pkg revisions is not 1 (was %s - %s)" % (pr_2, pr_1)) 92 self.assertTrue(pr_2 - pr_1 == 1, "New PR %s did not increment as expected (from %s), difference should be 1" % (pr_2, pr_1))
93 93
94 def test_import_export_replace_db(self): 94 def test_import_export_replace_db(self):
95 self.run_test_pr_export_import('m4') 95 self.run_test_pr_export_import('m4')