summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-08-31 23:44:42 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-09-01 15:51:11 +0100
commitc7b342903208cfb4cefaa0ec339c6ddfcd590acc (patch)
tree5fa96d0bb59baa99890548b684ce3e023982be90
parent6e15fee9eea00f754ad4a29a0d8586a215ba61f4 (diff)
downloadpoky-c7b342903208cfb4cefaa0ec339c6ddfcd590acc.tar.gz
bitbake: prserv/serv: Settle on two threads for optimal performance
Using the threading mixin class resulted in large amounts of memory being used by the PR server for no good reason. Using a receiver thread and a thread to do the actual database operations on a single connection gives the same performance with a much saner memory overhead so switch to this. (Bitbake rev: e08455d5f3b8e96765942b9c3b9767c30650557d) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/prserv/serv.py48
1 files changed, 34 insertions, 14 deletions
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 3677f77c80..a9c7ed104c 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -2,6 +2,8 @@ import os,sys,logging
2import signal, time, atexit, threading 2import signal, time, atexit, threading
3from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler 3from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
4import xmlrpclib 4import xmlrpclib
5import threading
6import Queue
5 7
6try: 8try:
7 import sqlite3 9 import sqlite3
@@ -31,14 +33,11 @@ class Handler(SimpleXMLRPCRequestHandler):
31PIDPREFIX = "/tmp/PRServer_%s_%s.pid" 33PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
32singleton = None 34singleton = None
33 35
34import SocketServer
35class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):
36 pass
37 36
38class PRServer(SimpleThreadedXMLRPCServer): 37class PRServer(SimpleXMLRPCServer):
39 def __init__(self, dbfile, logfile, interface, daemon=True): 38 def __init__(self, dbfile, logfile, interface, daemon=True):
40 ''' constructor ''' 39 ''' constructor '''
41 SimpleThreadedXMLRPCServer.__init__(self, interface, 40 SimpleXMLRPCServer.__init__(self, interface,
42 logRequests=False, allow_none=True) 41 logRequests=False, allow_none=True)
43 self.dbfile=dbfile 42 self.dbfile=dbfile
44 self.daemon=daemon 43 self.daemon=daemon
@@ -54,19 +53,41 @@ class PRServer(SimpleThreadedXMLRPCServer):
54 self.register_function(self.importone, "importone") 53 self.register_function(self.importone, "importone")
55 self.register_introspection_functions() 54 self.register_introspection_functions()
56 55
56 self.db = prserv.db.PRData(self.dbfile)
57 self.table = self.db["PRMAIN"]
58
59 self.requestqueue = Queue.Queue()
60 self.handlerthread = threading.Thread(target = self.process_request_thread)
61 self.handlerthread.daemon = False
62
63 def process_request_thread(self):
64 """Same as in BaseServer but as a thread.
65
66 In addition, exception handling is done here.
67
68 """
69 while True:
70 (request, client_address) = self.requestqueue.get()
71 try:
72 self.finish_request(request, client_address)
73 self.shutdown_request(request)
74 except:
75 self.handle_error(request, client_address)
76 self.shutdown_request(request)
77
78
79 def process_request(self, request, client_address):
80 self.requestqueue.put((request, client_address))
81
57 def export(self, version=None, pkgarch=None, checksum=None, colinfo=True): 82 def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
58 try: 83 try:
59 db = prserv.db.PRData(self.dbfile) 84 return self.table.export(version, pkgarch, checksum, colinfo)
60 table = db["PRMAIN"]
61 return table.export(version, pkgarch, checksum, colinfo)
62 except sqlite3.Error as exc: 85 except sqlite3.Error as exc:
63 logger.error(str(exc)) 86 logger.error(str(exc))
64 return None 87 return None
65 88
66 def importone(self, version, pkgarch, checksum, value): 89 def importone(self, version, pkgarch, checksum, value):
67 db = prserv.db.PRData(self.dbfile) 90 return self.table.importone(version, pkgarch, checksum, value)
68 table = db["PRMAIN"]
69 return table.importone(version, pkgarch, checksum, value)
70 91
71 def ping(self): 92 def ping(self):
72 return not self.quit 93 return not self.quit
@@ -76,9 +97,7 @@ class PRServer(SimpleThreadedXMLRPCServer):
76 97
77 def getPR(self, version, pkgarch, checksum): 98 def getPR(self, version, pkgarch, checksum):
78 try: 99 try:
79 db = prserv.db.PRData(self.dbfile) 100 return self.table.getValue(version, pkgarch, checksum)
80 table = db["PRMAIN"]
81 return table.getValue(version, pkgarch, checksum)
82 except prserv.NotFoundError: 101 except prserv.NotFoundError:
83 logger.error("can not find value for (%s, %s)",version, checksum) 102 logger.error("can not find value for (%s, %s)",version, checksum)
84 return None 103 return None
@@ -97,6 +116,7 @@ class PRServer(SimpleThreadedXMLRPCServer):
97 logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" % 116 logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
98 (self.dbfile, self.host, self.port, str(os.getpid()))) 117 (self.dbfile, self.host, self.port, str(os.getpid())))
99 118
119 self.handlerthread.start()
100 while not self.quit: 120 while not self.quit:
101 self.handle_request() 121 self.handle_request()
102 122