summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorHaris Okanovic <haris.okanovic@ni.com>2016-02-23 11:36:33 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-02-26 17:20:31 +0000
commit4a12865c7b8a534ac7f7c1380c549c5392eda5f5 (patch)
treead64bde4abc6136bf966418bb52551216764f829 /bitbake/lib
parentbdb51abb3ccedacb0c409b9ae702000170660f44 (diff)
downloadpoky-4a12865c7b8a534ac7f7c1380c549c5392eda5f5.tar.gz
bitbake: prserv: Add dump_db()
Returns a script (string) that reconstructs the state of the entire database at the time this function is called. The script language is defined by the backing database engine, which is a function of server configuration. Returns None if the database engine does not support dumping to script or if some other error is encountered in processing. The SQLite3 implementation in db.py calls iterdump() [1] to generate a script. iterdump() is the library equivalent of the `sqlite3 .dump` shell command, and the scripts are compatible. Execute the script in an empty SQLite3 database using the sqlite3 utility to restore a backup of prserv. Use case: Backup a live PR server database in a non-racy way, such that one could snapshot the entire database after a set of bitbake builds all using a shared server. I.e. All changes made prior to the start of a dump_db() operation should be committed and captured in the script. Subsequent changes made during the backup process are not guaranteed to be captured. Testing: ~7MB database backs up in ~1s while PR server is under load from 32 thread bitbake builds on two separate machines. [1] https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.iterdump (Bitbake rev: 004003daf6bd0f0233ce5c2d95f1d7d64ab91bb3) Signed-off-by: Haris Okanovic <haris.okanovic@ni.com> Reviewed-by: Ken Sharp <ken.sharp@ni.com> Reviewed-by: Bill Pittman <bill.pittman@ni.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/prserv/db.py8
-rw-r--r--bitbake/lib/prserv/serv.py25
2 files changed, 33 insertions, 0 deletions
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index 36c9f7b630..2a86184170 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -231,6 +231,14 @@ class PRTable(object):
231 datainfo.append(col) 231 datainfo.append(col)
232 return (metainfo, datainfo) 232 return (metainfo, datainfo)
233 233
234 def dump_db(self, fd):
235 writeCount = 0
236 for line in self.conn.iterdump():
237 writeCount = writeCount + len(line) + 1
238 fd.write(line)
239 fd.write('\n')
240 return writeCount
241
234class PRData(object): 242class PRData(object):
235 """Object representing the PR database""" 243 """Object representing the PR database"""
236 def __init__(self, filename, nohist=True): 244 def __init__(self, filename, nohist=True):
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index f588f4dc27..affccd9b3b 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -4,6 +4,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
4import threading 4import threading
5import Queue 5import Queue
6import socket 6import socket
7import StringIO
7 8
8try: 9try:
9 import sqlite3 10 import sqlite3
@@ -59,6 +60,7 @@ class PRServer(SimpleXMLRPCServer):
59 self.register_function(self.quit, "quit") 60 self.register_function(self.quit, "quit")
60 self.register_function(self.ping, "ping") 61 self.register_function(self.ping, "ping")
61 self.register_function(self.export, "export") 62 self.register_function(self.export, "export")
63 self.register_function(self.dump_db, "dump_db")
62 self.register_function(self.importone, "importone") 64 self.register_function(self.importone, "importone")
63 self.register_introspection_functions() 65 self.register_introspection_functions()
64 66
@@ -115,6 +117,26 @@ class PRServer(SimpleXMLRPCServer):
115 logger.error(str(exc)) 117 logger.error(str(exc))
116 return None 118 return None
117 119
120 def dump_db(self):
121 """
122 Returns a script (string) that reconstructs the state of the
123 entire database at the time this function is called. The script
124 language is defined by the backing database engine, which is a
125 function of server configuration.
126 Returns None if the database engine does not support dumping to
127 script or if some other error is encountered in processing.
128 """
129 buff = StringIO.StringIO()
130 try:
131 self.table.sync()
132 self.table.dump_db(buff)
133 return buff.getvalue()
134 except Exception as exc:
135 logger.error(str(exc))
136 return None
137 finally:
138 buff.close()
139
118 def importone(self, version, pkgarch, checksum, value): 140 def importone(self, version, pkgarch, checksum, value):
119 return self.table.importone(version, pkgarch, checksum, value) 141 return self.table.importone(version, pkgarch, checksum, value)
120 142
@@ -288,6 +310,9 @@ class PRServerConnection(object):
288 def export(self,version=None, pkgarch=None, checksum=None, colinfo=True): 310 def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
289 return self.connection.export(version, pkgarch, checksum, colinfo) 311 return self.connection.export(version, pkgarch, checksum, colinfo)
290 312
313 def dump_db(self):
314 return self.connection.dump_db()
315
291 def importone(self, version, pkgarch, checksum, value): 316 def importone(self, version, pkgarch, checksum, value):
292 return self.connection.importone(version, pkgarch, checksum, value) 317 return self.connection.importone(version, pkgarch, checksum, value)
293 318