summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorLianhao Lu <lianhao.lu@intel.com>2012-01-10 14:13:50 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-01-11 10:36:20 +0000
commit489cde8eb0e19ef6fe8078148199eaf5b52631ae (patch)
treecb85947000c8efb778362c136f1f6a0c14ed31dc /bitbake/lib
parent30a9bc6c92a8920d6e9c4a4b93b83bdbe5d48e78 (diff)
downloadpoky-489cde8eb0e19ef6fe8078148199eaf5b52631ae.tar.gz
bitbake: Automatically start local PR service.
[YOCTO #1126] A local PR service will be started and stopped automatically along with the bitbake invocation/ternimation. This local PR service will be started only and if only when the PRSERV_HOST is set to 'localhost' and PRSERV_PORT is set to '0'. When started, the sqlite3 database is stored at "${PERSISTEN_DIR}/prserv.sqlite3" or "${CACHE}/prserv.sqlite3". (Bitbake rev: 9d8f45407c67ed0d3c4f820cf646de3c385067c7) Signed-off-by: Lianhao Lu <lianhao.lu@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/cooker.py3
-rw-r--r--bitbake/lib/prserv/db.py12
-rw-r--r--bitbake/lib/prserv/serv.py144
3 files changed, 119 insertions, 40 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 203271868a..194046ea91 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -36,6 +36,7 @@ from functools import wraps
36from collections import defaultdict 36from collections import defaultdict
37import bb, bb.exceptions, bb.command 37import bb, bb.exceptions, bb.command
38from bb import utils, data, parse, event, cache, providers, taskdata, runqueue 38from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
39import prserv.serv
39 40
40logger = logging.getLogger("BitBake") 41logger = logging.getLogger("BitBake")
41collectlog = logging.getLogger("BitBake.Collection") 42collectlog = logging.getLogger("BitBake.Collection")
@@ -1311,9 +1312,11 @@ class BBCooker:
1311 # Empty the environment. The environment will be populated as 1312 # Empty the environment. The environment will be populated as
1312 # necessary from the data store. 1313 # necessary from the data store.
1313 #bb.utils.empty_environment() 1314 #bb.utils.empty_environment()
1315 prserv.serv.auto_start(self.configuration.data)
1314 return 1316 return
1315 1317
1316 def post_serve(self): 1318 def post_serve(self):
1319 prserv.serv.auto_shutdown(self.configuration.data)
1317 bb.event.fire(CookerExit(), self.configuration.event_data) 1320 bb.event.fire(CookerExit(), self.configuration.event_data)
1318 1321
1319 def shutdown(self): 1322 def shutdown(self):
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index f267daed13..9d8e9db9f2 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -8,6 +8,8 @@ try:
8except ImportError: 8except ImportError:
9 from pysqlite2 import dbapi2 as sqlite3 9 from pysqlite2 import dbapi2 as sqlite3
10 10
11logger = logging.getLogger("BitBake.PRserv")
12
11sqlversion = sqlite3.sqlite_version_info 13sqlversion = sqlite3.sqlite_version_info
12if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): 14if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
13 raise Exception("sqlite3 version 3.3.0 or later is required.") 15 raise Exception("sqlite3 version 3.3.0 or later is required.")
@@ -55,7 +57,7 @@ class PRTable():
55 (version,pkgarch, checksum,version, pkgarch)) 57 (version,pkgarch, checksum,version, pkgarch))
56 self.conn.commit() 58 self.conn.commit()
57 except sqlite3.IntegrityError as exc: 59 except sqlite3.IntegrityError as exc:
58 logging.error(str(exc)) 60 logger.error(str(exc))
59 61
60 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 62 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
61 (version, pkgarch, checksum)) 63 (version, pkgarch, checksum))
@@ -83,7 +85,7 @@ class PRTable():
83 (version, pkgarch, checksum, version, pkgarch)) 85 (version, pkgarch, checksum, version, pkgarch))
84 self.conn.commit() 86 self.conn.commit()
85 except sqlite3.IntegrityError as exc: 87 except sqlite3.IntegrityError as exc:
86 logging.error(str(exc)) 88 logger.error(str(exc))
87 self.conn.rollback() 89 self.conn.rollback()
88 90
89 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 91 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
@@ -115,7 +117,7 @@ class PRTable():
115 (version, pkgarch, checksum, value)) 117 (version, pkgarch, checksum, value))
116 self.conn.commit() 118 self.conn.commit()
117 except sqlite3.IntegrityError as exc: 119 except sqlite3.IntegrityError as exc:
118 logging.error(str(exc)) 120 logger.error(str(exc))
119 121
120 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 122 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
121 (version, pkgarch, checksum)) 123 (version, pkgarch, checksum))
@@ -140,7 +142,7 @@ class PRTable():
140 (value,version,pkgarch,checksum,value)) 142 (value,version,pkgarch,checksum,value))
141 self.conn.commit() 143 self.conn.commit()
142 except sqlite3.IntegrityError as exc: 144 except sqlite3.IntegrityError as exc:
143 logging.error(str(exc)) 145 logger.error(str(exc))
144 146
145 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, 147 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
146 (version,pkgarch,checksum,value)) 148 (version,pkgarch,checksum,value))
@@ -241,5 +243,5 @@ class PRData(object):
241 def __delitem__(self, tblname): 243 def __delitem__(self, tblname):
242 if tblname in self._tables: 244 if tblname in self._tables:
243 del self._tables[tblname] 245 del self._tables[tblname]
244 logging.info("drop table %s" % (tblname)) 246 logger.info("drop table %s" % (tblname))
245 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 247 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 7bcffa7744..a759fa7417 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -1,5 +1,5 @@
1import os,sys,logging 1import os,sys,logging
2import signal,time, atexit 2import signal, time, atexit, threading
3from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler 3from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
4import xmlrpclib,sqlite3 4import xmlrpclib,sqlite3
5 5
@@ -7,6 +7,8 @@ import bb.server.xmlrpc
7import prserv 7import prserv
8import prserv.db 8import prserv.db
9 9
10logger = logging.getLogger("BitBake.PRserv")
11
10if sys.hexversion < 0x020600F0: 12if sys.hexversion < 0x020600F0:
11 print("Sorry, python 2.6 or later is required.") 13 print("Sorry, python 2.6 or later is required.")
12 sys.exit(1) 14 sys.exit(1)
@@ -22,9 +24,9 @@ class Handler(SimpleXMLRPCRequestHandler):
22 return value 24 return value
23 25
24PIDPREFIX = "/tmp/PRServer_%s_%s.pid" 26PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
27singleton = None
25 28
26class PRServer(SimpleXMLRPCServer): 29class PRServer(SimpleXMLRPCServer):
27 pidfile="/tmp/PRServer.pid"
28 def __init__(self, dbfile, logfile, interface, daemon=True): 30 def __init__(self, dbfile, logfile, interface, daemon=True):
29 ''' constructor ''' 31 ''' constructor '''
30 SimpleXMLRPCServer.__init__(self, interface, 32 SimpleXMLRPCServer.__init__(self, interface,
@@ -33,10 +35,11 @@ class PRServer(SimpleXMLRPCServer):
33 self.dbfile=dbfile 35 self.dbfile=dbfile
34 self.daemon=daemon 36 self.daemon=daemon
35 self.logfile=logfile 37 self.logfile=logfile
38 self.working_thread=None
36 self.host, self.port = self.socket.getsockname() 39 self.host, self.port = self.socket.getsockname()
37 self.db=prserv.db.PRData(dbfile) 40 self.db=prserv.db.PRData(dbfile)
38 self.table=self.db["PRMAIN"] 41 self.table=self.db["PRMAIN"]
39 self.pidfile=PIDPREFIX % interface 42 self.pidfile=PIDPREFIX % (self.host, self.port)
40 43
41 self.register_function(self.getPR, "getPR") 44 self.register_function(self.getPR, "getPR")
42 self.register_function(self.quit, "quit") 45 self.register_function(self.quit, "quit")
@@ -44,12 +47,12 @@ class PRServer(SimpleXMLRPCServer):
44 self.register_function(self.export, "export") 47 self.register_function(self.export, "export")
45 self.register_function(self.importone, "importone") 48 self.register_function(self.importone, "importone")
46 self.register_introspection_functions() 49 self.register_introspection_functions()
47 50
48 def export(self, version=None, pkgarch=None, checksum=None, colinfo=True): 51 def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
49 try: 52 try:
50 return self.table.export(version, pkgarch, checksum, colinfo) 53 return self.table.export(version, pkgarch, checksum, colinfo)
51 except sqlite3.Error as exc: 54 except sqlite3.Error as exc:
52 logging.error(str(exc)) 55 logger.error(str(exc))
53 return None 56 return None
54 57
55 def importone(self, version, pkgarch, checksum, value): 58 def importone(self, version, pkgarch, checksum, value):
@@ -58,45 +61,47 @@ class PRServer(SimpleXMLRPCServer):
58 def ping(self): 61 def ping(self):
59 return not self.quit 62 return not self.quit
60 63
64 def getinfo(self):
65 return (self.host, self.port)
66
61 def getPR(self, version, pkgarch, checksum): 67 def getPR(self, version, pkgarch, checksum):
62 try: 68 try:
63 return self.table.getValue(version, pkgarch, checksum) 69 return self.table.getValue(version, pkgarch, checksum)
64 except prserv.NotFoundError: 70 except prserv.NotFoundError:
65 logging.error("can not find value for (%s, %s)",version, checksum) 71 logger.error("can not find value for (%s, %s)",version, checksum)
66 return None 72 return None
67 except sqlite3.Error as exc: 73 except sqlite3.Error as exc:
68 logging.error(str(exc)) 74 logger.error(str(exc))
69 return None 75 return None
70 76
71 def quit(self): 77 def quit(self):
72 self.quit=True 78 self.quit=True
73 return 79 return
74 80
75 def _serve_forever(self): 81 def work_forever(self,):
76 self.quit = False 82 self.quit = False
77 self.timeout = 0.5 83 self.timeout = 0.5
84 logger.info("PRServer: started! DBfile: %s, IP: %s, PORT: %s, PID: %s" %
85 (self.dbfile, self.host, self.port, str(os.getpid())))
86
78 while not self.quit: 87 while not self.quit:
79 self.handle_request() 88 self.handle_request()
80 89
81 logging.info("PRServer: stopping...") 90 logger.info("PRServer: stopping...")
82 self.server_close() 91 self.server_close()
83 return 92 return
84 93
85 def start(self): 94 def start(self):
86 if self.daemon is True: 95 if self.daemon is True:
87 logging.info("PRServer: try to start daemon...") 96 logger.info("PRServer: try to start daemon...")
88 self.daemonize() 97 self.daemonize()
89 else: 98 else:
90 atexit.register(self.delpid) 99 atexit.register(self.delpid)
91 pid = str(os.getpid()) 100 pid = str(os.getpid())
92 pf = file(self.pidfile, 'w+') 101 pf = file(self.pidfile, 'w+')
93 pf.write("%s\n" % pid) 102 pf.write("%s\n" % pid)
94 pf.write("%s\n" % self.host)
95 pf.write("%s\n" % self.port)
96 pf.close() 103 pf.close()
97 logging.info("PRServer: start success! DBfile: %s, IP: %s, PORT: %d" % 104 self.work_forever()
98 (self.dbfile, self.host, self.port))
99 self._serve_forever()
100 105
101 def delpid(self): 106 def delpid(self):
102 os.remove(self.pidfile) 107 os.remove(self.pidfile)
@@ -144,17 +149,40 @@ class PRServer(SimpleXMLRPCServer):
144 pf.write("%s\n" % pid) 149 pf.write("%s\n" % pid)
145 pf.close() 150 pf.close()
146 151
147 logging.info("PRServer: starting daemon success! DBfile: %s, IP: %s, PORT: %s, PID: %s" % 152 self.work_forever()
148 (self.dbfile, self.host, self.port, pid)) 153 sys.exit(0)
154
155class PRServSingleton():
156 def __init__(self, dbfile, logfile, interface):
157 self.dbfile = dbfile
158 self.logfile = logfile
159 self.interface = interface
160 self.host = None
161 self.port = None
162 self.event = threading.Event()
163
164 def _work(self):
165 self.prserv = PRServer(self.dbfile, self.logfile, self.interface, False)
166 self.host, self.port = self.prserv.getinfo()
167 self.event.set()
168 self.prserv.work_forever()
169 del self.prserv.db
170
171 def start(self):
172 self.working_thread = threading.Thread(target=self._work)
173 self.working_thread.start()
149 174
150 self._serve_forever() 175 def getinfo(self):
151 exit(0) 176 self.event.wait()
177 return (self.host, self.port)
152 178
153class PRServerConnection(): 179class PRServerConnection():
154 def __init__(self, host, port): 180 def __init__(self, host, port):
155 self.connection = bb.server.xmlrpc._create_server(host, port) 181 if is_local_special(host, port):
182 host, port = singleton.getinfo()
156 self.host = host 183 self.host = host
157 self.port = port 184 self.port = port
185 self.connection = bb.server.xmlrpc._create_server(self.host, self.port)
158 186
159 def terminate(self): 187 def terminate(self):
160 # Don't wait for server indefinitely 188 # Don't wait for server indefinitely
@@ -173,13 +201,14 @@ class PRServerConnection():
173 201
174 def export(self,version=None, pkgarch=None, checksum=None, colinfo=True): 202 def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
175 return self.connection.export(version, pkgarch, checksum, colinfo) 203 return self.connection.export(version, pkgarch, checksum, colinfo)
176 204
177 def importone(self, version, pkgarch, checksum, value): 205 def importone(self, version, pkgarch, checksum, value):
178 return self.connection.importone(version, pkgarch, checksum, value) 206 return self.connection.importone(version, pkgarch, checksum, value)
179 207
180def start_daemon(dbfile, logfile, interface): 208def start_daemon(dbfile, host, port, logfile):
209 pidfile = PIDPREFIX % (host, port)
181 try: 210 try:
182 pf = file(PRServer.pidfile,'r') 211 pf = file(pidfile,'r')
183 pid = int(pf.readline().strip()) 212 pid = int(pf.readline().strip())
184 pf.close() 213 pf.close()
185 except IOError: 214 except IOError:
@@ -187,10 +216,10 @@ def start_daemon(dbfile, logfile, interface):
187 216
188 if pid: 217 if pid:
189 sys.stderr.write("pidfile %s already exist. Daemon already running?\n" 218 sys.stderr.write("pidfile %s already exist. Daemon already running?\n"
190 % PRServer.pidfile) 219 % pidfile)
191 return 1 220 return 1
192 221
193 server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), interface) 222 server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), (host,port))
194 server.start() 223 server.start()
195 return 0 224 return 0
196 225
@@ -206,25 +235,70 @@ def stop_daemon(host, port):
206 if not pid: 235 if not pid:
207 sys.stderr.write("pidfile %s does not exist. Daemon not running?\n" 236 sys.stderr.write("pidfile %s does not exist. Daemon not running?\n"
208 % pidfile) 237 % pidfile)
209 return 1
210 238
211 PRServerConnection(host, port).terminate() 239 try:
240 PRServerConnection(host, port).terminate()
241 except:
242 logger.critical("Stop PRService %s:%d failed" % (host,port))
212 time.sleep(0.5) 243 time.sleep(0.5)
213 244
214 try: 245 try:
215 while 1: 246 if pid:
247 if os.path.exists(pidfile):
248 os.remove(pidfile)
216 os.kill(pid,signal.SIGTERM) 249 os.kill(pid,signal.SIGTERM)
217 time.sleep(0.1) 250 time.sleep(0.1)
218 except OSError as e: 251 except OSError as e:
219 err = str(e) 252 err = str(e)
220 if err.find("No such process") > 0: 253 if err.find("No such process") <= 0:
221 if os.path.exists(PRServer.pidfile): 254 raise e
222 os.remove(PRServer.pidfile)
223 else:
224 raise Exception("%s [%d]" % (e.strerror, e.errno))
225 255
226 return 0 256 return 0
227 257
258def is_local_special(host, port):
259 if host.strip().upper() == 'localhost'.upper() and (not port):
260 return True
261 else:
262 return False
263
264def auto_start(d):
265 global singleton
266 if d.getVar('USE_PR_SERV', True) == '0':
267 return True
268
269 if is_local_special(d.getVar('PRSERV_HOST', True), int(d.getVar('PRSERV_PORT', True))) and not singleton:
270 import bb.utils
271 cachedir = (d.getVar("PERSISTENT_DIR", True) or d.getVar("CACHE", True))
272 if not cachedir:
273 logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
274 sys.exit(1)
275 bb.utils.mkdirhier(cachedir)
276 dbfile = os.path.join(cachedir, "prserv.sqlite3")
277 logfile = os.path.join(cachedir, "prserv.log")
278 singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), ("localhost",0))
279 singleton.start()
280 if singleton:
281 host, port = singleton.getinfo()
282 else:
283 host = d.getVar('PRSERV_HOST', True)
284 port = int(d.getVar('PRSERV_PORT', True))
285
286 try:
287 return PRServerConnection(host,port).ping()
288 except Exception:
289 logger.critical("PRservice %s:%d not available" % (host, port))
290 return False
291
292def auto_shutdown(d=None):
293 global singleton
294 if singleton:
295 host, port = singleton.getinfo()
296 try:
297 PRServerConnection(host, port).terminate()
298 except:
299 logger.critical("Stop PRService %s:%d failed" % (host,port))
300 singleton = None
301
228def ping(host, port): 302def ping(host, port):
229 print PRServerConnection(host,port).ping() 303 conn=PRServerConnection(host, port)
230 return 0 304 return conn.ping()