summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLianhao Lu <lianhao.lu@intel.com>2011-12-22 15:29:11 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-01-11 10:37:43 +0000
commita05e3a57c6f567578faeb31fae89b20e22850af4 (patch)
tree0fb66b3c74a9c011190179f0004024be5927570c
parent6bde156c5ce58c8e55cd6f5fd5f87982a919ba34 (diff)
downloadpoky-a05e3a57c6f567578faeb31fae89b20e22850af4.tar.gz
meta/PRService: Added export/import fuctions.
[YOCTO #1556] - Modified meta/class/package.bbclass and prserv.bbclass according to the change in PR service by adding PACKAGE_ARCH into the query tuple. - Added prexport.bbclass, primport.bbclass to export/import AUTOPR values from/to PRService. - Move PR service related common code to lib/oe/prservice.py. - Supported reading the AUTOPR values from the exported .inc file instead of reading it from remote PR service. - Created a new script bitbake-prserv-tool to export/import the AUTOPR values from/to the PR service. Typical usage scenario of the export/import is: 1. bitbake-prserv-tool export <file> to export the AUTOPR values from the current PR service into an exported .inc file. 2. Others may use that exported .inc file(to be included in the local.conf) to lockdown and reproduce the same AUTOPR when generating package feeds. 3. Others may "bitbake-prserv-tool import <file>" to import the AUTOPR values into their own PR service and the AUTOPR values will be incremented from there. (From OE-Core rev: 9979107d8eaf503efd921564385859b1e83dbb3c) Signed-off-by: Lianhao Lu <lianhao.lu@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/package.bbclass13
-rw-r--r--meta/classes/prexport.bbclass45
-rw-r--r--meta/classes/primport.bbclass17
-rw-r--r--meta/classes/prserv.bbclass36
-rw-r--r--meta/conf/bitbake.conf4
-rw-r--r--meta/conf/prexport.conf1
-rw-r--r--meta/conf/primport.conf1
-rw-r--r--meta/lib/oe/prservice.py113
-rwxr-xr-xscripts/bitbake-prserv-tool57
9 files changed, 260 insertions, 27 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 9040eb40ba..65e65715ea 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -351,10 +351,17 @@ def runtime_mapping_rename (varname, d):
351 351
352python package_get_auto_pr() { 352python package_get_auto_pr() {
353 if d.getVar('USE_PR_SERV', True) != "0": 353 if d.getVar('USE_PR_SERV', True) != "0":
354 auto_pr=prserv_get_pr_auto(d) 354 try:
355 if auto_pr is None: 355 auto_pr=prserv_get_pr_auto(d)
356 bb.fatal("Can NOT get auto PR revision from remote PR service") 356 except Exception as e:
357 bb.fatal("Can NOT get PRAUTO, exception %s" % str(e))
357 return 358 return
359 if auto_pr is None:
360 if d.getVar('PRSERV_LOCKDOWN', True):
361 bb.fatal("Can NOT get PRAUTO from lockdown exported file")
362 else:
363 bb.fatal("Can NOT get PRAUTO from remote PR service")
364 return
358 d.setVar('PRAUTO',str(auto_pr)) 365 d.setVar('PRAUTO',str(auto_pr))
359} 366}
360 367
diff --git a/meta/classes/prexport.bbclass b/meta/classes/prexport.bbclass
new file mode 100644
index 0000000000..5b5b707a88
--- /dev/null
+++ b/meta/classes/prexport.bbclass
@@ -0,0 +1,45 @@
1PRSERV_DUMPOPT_VERSION = "${PRAUTOINX}"
2PRSERV_DUMPOPT_PKGARCH = ""
3PRSERV_DUMPOPT_CHECKSUM = ""
4PRSERV_DUMPOPT_COL = "0"
5
6PRSERV_DUMPDIR ??= "${LOG_DIR}/db"
7PRSERV_DUMPFILE ??= "${PRSERV_DUMPDIR}/prserv.inc"
8
9python prexport_handler () {
10 import bb.event
11 if not e.data:
12 return
13
14 if isinstance(e, bb.event.RecipeParsed):
15 import oe.prservice
16 #get all PR values for the current PRAUTOINX
17 ver = e.data.getVar('PRSERV_DUMPOPT_VERSION', True)
18 ver = ver.replace('%','-')
19 retval = oe.prservice.prserv_dump_db(e.data)
20 if not retval:
21 bb.fatal("prexport_handler: export failed!")
22 (metainfo, datainfo) = retval
23 if not datainfo:
24 bb.error("prexport_handler: No AUROPR values found for %s" % ver)
25 return
26 oe.prservice.prserv_export_tofile(e.data, None, datainfo, False)
27 elif isinstance(e, bb.event.ParseStarted):
28 import bb.utils
29 #remove dumpfile
30 bb.utils.remove(e.data.getVar('PRSERV_DUMPFILE', True))
31 elif isinstance(e, bb.event.ParseCompleted):
32 import oe.prservice
33 #dump meta info of tables
34 d = e.data.createCopy()
35 d.setVar('PRSERV_DUMPOPT_COL', "1")
36 retval = oe.prservice.prserv_dump_db(d)
37 if not retval:
38 bb.error("prexport_handler: export failed!")
39 return
40 (metainfo, datainfo) = retval
41 oe.prservice.prserv_export_tofile(d, metainfo, None, True)
42
43}
44
45addhandler prexport_handler
diff --git a/meta/classes/primport.bbclass b/meta/classes/primport.bbclass
new file mode 100644
index 0000000000..08e5a8f426
--- /dev/null
+++ b/meta/classes/primport.bbclass
@@ -0,0 +1,17 @@
1python primport_handler () {
2 import bb.event
3 if not e.data:
4 return
5
6 if isinstance(e, bb.event.ParseCompleted):
7 import oe.prservice
8 #import all exported AUTOPR values
9 imported = oe.prservice.prserv_import_db(e.data)
10 if imported is None:
11 bb.fatal("import failed!")
12
13 for (version, pkgarch, checksum, value) in imported:
14 bb.note("imported (%s,%s,%s,%d)" % (version, pkgarch, checksum, value))
15}
16
17addhandler primport_handler
diff --git a/meta/classes/prserv.bbclass b/meta/classes/prserv.bbclass
index 18b8589a7b..0825306f91 100644
--- a/meta/classes/prserv.bbclass
+++ b/meta/classes/prserv.bbclass
@@ -1,29 +1,21 @@
1def prserv_make_conn(d):
2 import prserv.serv
3 host=d.getVar("PRSERV_HOST",True)
4 port=d.getVar("PRSERV_PORT",True)
5 try:
6 conn=None
7 conn=prserv.serv.PRServerConnection(host,int(port))
8 d.setVar("__PRSERV_CONN",conn)
9 except Exception, exc:
10 bb.fatal("Connecting to PR service %s:%s failed: %s" % (host, port, str(exc)))
11
12 return conn
13
14def prserv_get_pr_auto(d): 1def prserv_get_pr_auto(d):
15 if d.getVar('USE_PR_SERV', True) != "0": 2 import oe.prservice
3 if d.getVar('USE_PR_SERV', True) != "1":
16 bb.warn("Not using network based PR service") 4 bb.warn("Not using network based PR service")
17 return None 5 return None
18 6
19 conn=d.getVar("__PRSERV_CONN", True) 7 version = d.getVar("PRAUTOINX", True)
20 if conn is None: 8 pkgarch = d.getVar("PACKAGE_ARCH", True)
21 conn=prserv_make_conn(d) 9 checksum = d.getVar("BB_TASKHASH", True)
10
11 if d.getVar('PRSERV_LOCKDOWN', True):
12 auto_rev = d.getVar('PRAUTO_' + version + '_' + pkgarch, True) or d.getVar('PRAUTO_' + version, True) or None
13 else:
14 conn = d.getVar("__PRSERV_CONN", True)
22 if conn is None: 15 if conn is None:
23 return None 16 conn = oe.prservice.prserv_make_conn(d)
17 if conn is None:
18 return None
19 auto_rev = conn.getPR(version, pkgarch, checksum)
24 20
25 version=d.getVar("PF", True)
26 checksum=d.getVar("BB_TASKHASH", True)
27 auto_rev=conn.getPR(version,checksum)
28 bb.debug(1,"prserv_get_pr_auto: version: %s checksum: %s result %d" % (version, checksum, auto_rev))
29 return auto_rev 21 return auto_rev
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 336b1d7062..43eedad046 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -190,7 +190,7 @@ BP = "${BPN}-${PV}"
190# 190#
191# network based PR service 191# network based PR service
192# 192#
193USE_PR_SERV = "${@[1,0][(d.getVar('PRSERV_HOST',1) is None) or (d.getVar('PRSERV_PORT',1) is None)]}" 193USE_PR_SERV = "${@[1,0][((not d.getVar('PRSERV_HOST', True)) or (not d.getVar('PRSERV_PORT', True))) and (not d.getVar('PRSERV_LOCKDOWN', True))]}"
194 194
195# Package info. 195# Package info.
196 196
@@ -732,7 +732,7 @@ BB_CONSOLELOG = "${TMPDIR}/cooker.log.${DATETIME}"
732# Setup our default hash policy 732# Setup our default hash policy
733BB_SIGNATURE_HANDLER ?= "basic" 733BB_SIGNATURE_HANDLER ?= "basic"
734BB_HASHTASK_WHITELIST ?= "(.*-cross$|.*-native$|.*-cross-initial$|.*-cross-intermediate$|^virtual:native:.*|^virtual:nativesdk:.*)" 734BB_HASHTASK_WHITELIST ?= "(.*-cross$|.*-native$|.*-cross-initial$|.*-cross-intermediate$|^virtual:native:.*|^virtual:nativesdk:.*)"
735BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE" 735BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST PRSERV_PORT PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN"
736 736
737MLPREFIX ??= "" 737MLPREFIX ??= ""
738MULTILIB_VARIANTS ??= "" 738MULTILIB_VARIANTS ??= ""
diff --git a/meta/conf/prexport.conf b/meta/conf/prexport.conf
new file mode 100644
index 0000000000..12f3acb2da
--- /dev/null
+++ b/meta/conf/prexport.conf
@@ -0,0 +1 @@
INHERIT += "prexport"
diff --git a/meta/conf/primport.conf b/meta/conf/primport.conf
new file mode 100644
index 0000000000..d94ea1b1e2
--- /dev/null
+++ b/meta/conf/primport.conf
@@ -0,0 +1 @@
INHERIT += "primport"
diff --git a/meta/lib/oe/prservice.py b/meta/lib/oe/prservice.py
new file mode 100644
index 0000000000..fa6718e914
--- /dev/null
+++ b/meta/lib/oe/prservice.py
@@ -0,0 +1,113 @@
1import bb
2
3def prserv_make_conn(d):
4 import prserv.serv
5 host = d.getVar("PRSERV_HOST",True)
6 port = d.getVar("PRSERV_PORT",True)
7 try:
8 conn = None
9 conn = prserv.serv.PRServerConnection(host,int(port))
10 d.setVar("__PRSERV_CONN",conn)
11 except Exception, exc:
12 bb.fatal("Connecting to PR service %s:%s failed: %s" % (host, port, str(exc)))
13
14 return conn
15
16def prserv_dump_db(d):
17 if d.getVar('USE_PR_SERV', True) != "1":
18 bb.error("Not using network based PR service")
19 return None
20
21 conn = d.getVar("__PRSERV_CONN", True)
22 if conn is None:
23 conn = prserv_make_conn(d)
24 if conn is None:
25 bb.error("Making connection failed to remote PR service")
26 return None
27
28 #dump db
29 opt_version = d.getVar('PRSERV_DUMPOPT_VERSION', True)
30 opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH', True)
31 opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM', True)
32 opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL', True))
33 return conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col)
34
35def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None):
36 if d.getVar('USE_PR_SERV', True) != "1":
37 bb.error("Not using network based PR service")
38 return None
39
40 conn = d.getVar("__PRSERV_CONN", True)
41 if conn is None:
42 conn = prserv_make_conn(d)
43 if conn is None:
44 bb.error("Making connection failed to remote PR service")
45 return None
46 #get the entry values
47 imported = []
48 prefix = "PRAUTO$"
49 for v in d.keys():
50 if v.startswith(prefix):
51 (remain, sep, checksum) = v.rpartition('$')
52 (remain, sep, pkgarch) = remain.rpartition('$')
53 (remain, sep, version) = remain.rpartition('$')
54 if (remain + '$' != prefix) or \
55 (filter_version and filter_version != version) or \
56 (filter_pkgarch and filter_pkgarch != pkgarch) or \
57 (filter_checksum and filter_checksum != checksum):
58 continue
59 try:
60 value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum, True))
61 except BaseException as exc:
62 bb.debug("Not valid value of %s:%s" % (v,str(exc)))
63 continue
64 ret = conn.importone(version,pkgarch,checksum,value)
65 if ret != value:
66 bb.error("importing(%s,%s,%s,%d) failed. DB may have larger value %d" % (version,pkgarch,checksum,value,ret))
67 else:
68 imported.append((version,pkgarch,checksum,value))
69 return imported
70
71def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False):
72 import bb.utils
73 #initilize the output file
74 bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR', True))
75 df = d.getVar('PRSERV_DUMPFILE', True)
76 #write data
77 lf = bb.utils.lockfile("%s.lock" % df)
78 f = open(df, "a")
79 if metainfo:
80 #dump column info
81 f.write("#PR_core_ver = \"%s\"\n\n" % metainfo['core_ver']);
82 f.write("#Table: %s\n" % metainfo['tbl_name'])
83 f.write("#Columns:\n")
84 f.write("#name \t type \t notn \t dflt \t pk\n")
85 f.write("#----------\t --------\t --------\t --------\t ----\n")
86 for i in range(len(metainfo['col_info'])):
87 f.write("#%10s\t %8s\t %8s\t %8s\t %4s\n" %
88 (metainfo['col_info'][i]['name'],
89 metainfo['col_info'][i]['type'],
90 metainfo['col_info'][i]['notnull'],
91 metainfo['col_info'][i]['dflt_value'],
92 metainfo['col_info'][i]['pk']))
93 f.write("\n")
94
95 if lockdown:
96 f.write("PRSERV_LOCKDOWN = \"1\"\n\n")
97
98 if datainfo:
99 idx = {}
100 for i in range(len(datainfo)):
101 pkgarch = datainfo[i]['pkgarch']
102 value = datainfo[i]['value']
103 if not idx.has_key(pkgarch):
104 idx[pkgarch] = i
105 elif value > datainfo[idx[pkgarch]]['value']:
106 idx[pkgarch] = i
107 f.write("PRAUTO$%s$%s$%s = \"%s\"\n" %
108 (str(datainfo[i]['version']), pkgarch, str(datainfo[i]['checksum']), str(value)))
109 if not nomax:
110 for i in idx:
111 f.write("PRAUTO_%s_%s = \"%s\"\n" % (str(datainfo[idx[i]]['version']),str(datainfo[idx[i]]['pkgarch']),str(datainfo[idx[i]]['value'])))
112 f.close()
113 bb.utils.unlockfile(lf)
diff --git a/scripts/bitbake-prserv-tool b/scripts/bitbake-prserv-tool
new file mode 100755
index 0000000000..6c0584c01e
--- /dev/null
+++ b/scripts/bitbake-prserv-tool
@@ -0,0 +1,57 @@
1#!/usr/bin/env bash
2
3help ()
4{
5 base=`basename $0`
6 echo -e "Usage: $base command"
7 echo "Avaliable commands:"
8 echo -e "\texport <file>: export and lock down the AUTOPR values from the PR service into a file for release."
9 echo -e "\timport <file>: import the AUTOPR values from the exported file into the PR service."
10}
11
12export ()
13{
14 file=$1
15 [ "x${file}" == "x" ] && help && exit 1
16 rm -f ${file}
17
18 touch dummy.inc
19 bitbake -R conf/prexport.conf -R dummy.inc -p
20 s=`bitbake -R conf/prexport.conf -R dummy.inc -e | grep ^PRSERV_DUMPFILE= | cut -f2 -d\"`
21 rm -f dummy.inc
22 if [ "x${s}" != "x" ];
23 then
24 [ -e $s ] && mv -f $s $file && echo "Exporting to file $file succeeded!"
25 return 0
26 fi
27 echo "Exporting to file $file failed!"
28 return 1
29}
30
31import ()
32{
33 file=$1
34 [ "x${file}" == "x" ] && help && exit 1
35
36 touch dummy.inc
37 bitbake -R conf/primport.conf -R dummy.inc -R $file -p
38 ret=$?
39 rm -f dummy.inc
40 [ $ret -eq 0 ] && echo "Importing from file $file succeeded!" || echo "Importing from file $file failed!"
41 return $ret
42}
43
44[ $# -eq 0 ] && help && exit 1
45
46case $1 in
47export)
48 export $2
49 ;;
50import)
51 import $2
52 ;;
53*)
54 help
55 exit 1
56 ;;
57esac