summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBen Shelton <ben.shelton@ni.com>2014-10-27 12:27:23 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-10-28 15:00:39 +0000
commitcf82c6756bad8f7a5edc92ce513e02816b88efb3 (patch)
treea354f97b4e038ef0af88f60651a9a56926e3143e
parent2794f916ff4f13f71ecd829c2f0a838634d35352 (diff)
downloadpoky-cf82c6756bad8f7a5edc92ce513e02816b88efb3.tar.gz
bitbake: prserv: don't wait until exit to sync
In the commit 'prserv: Ensure data is committed', the PR server moved to only committing transactions to the database when the PR server is stopped. This improves performance, but it means that if the machine running the PR server loses power unexpectedly or if the PR server process gets SIGKILL, the uncommitted package revision data is lost. To fix this issue, sync the database periodically, once per 30 seconds by default, if it has been marked as dirty. To be safe, continue to sync the database at exit regardless of its status. (Bitbake rev: 424df64f2e9679043f0ce2b4f7dfc59c3d404304) Signed-off-by: Ben Shelton <ben.shelton@ni.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/prserv/db.py14
-rw-r--r--bitbake/lib/prserv/serv.py7
2 files changed, 21 insertions, 0 deletions
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index 49f36da1ad..3bdc04692e 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -19,6 +19,7 @@ class PRTable(object):
19 def __init__(self, conn, table, nohist): 19 def __init__(self, conn, table, nohist):
20 self.conn = conn 20 self.conn = conn
21 self.nohist = nohist 21 self.nohist = nohist
22 self.dirty = False
22 if nohist: 23 if nohist:
23 self.table = "%s_nohist" % table 24 self.table = "%s_nohist" % table
24 else: 25 else:
@@ -47,6 +48,11 @@ class PRTable(object):
47 self.conn.commit() 48 self.conn.commit()
48 self._execute("BEGIN EXCLUSIVE TRANSACTION") 49 self._execute("BEGIN EXCLUSIVE TRANSACTION")
49 50
51 def sync_if_dirty(self):
52 if self.dirty:
53 self.sync()
54 self.dirty = False
55
50 def _getValueHist(self, version, pkgarch, checksum): 56 def _getValueHist(self, version, pkgarch, checksum):
51 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 57 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
52 (version, pkgarch, checksum)) 58 (version, pkgarch, checksum))
@@ -62,6 +68,8 @@ class PRTable(object):
62 except sqlite3.IntegrityError as exc: 68 except sqlite3.IntegrityError as exc:
63 logger.error(str(exc)) 69 logger.error(str(exc))
64 70
71 self.dirty = True
72
65 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 73 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
66 (version, pkgarch, checksum)) 74 (version, pkgarch, checksum))
67 row=data.fetchone() 75 row=data.fetchone()
@@ -89,6 +97,8 @@ class PRTable(object):
89 logger.error(str(exc)) 97 logger.error(str(exc))
90 self.conn.rollback() 98 self.conn.rollback()
91 99
100 self.dirty = True
101
92 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 102 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
93 (version, pkgarch, checksum)) 103 (version, pkgarch, checksum))
94 row=data.fetchone() 104 row=data.fetchone()
@@ -118,6 +128,8 @@ class PRTable(object):
118 except sqlite3.IntegrityError as exc: 128 except sqlite3.IntegrityError as exc:
119 logger.error(str(exc)) 129 logger.error(str(exc))
120 130
131 self.dirty = True
132
121 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 133 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
122 (version, pkgarch, checksum)) 134 (version, pkgarch, checksum))
123 row = data.fetchone() 135 row = data.fetchone()
@@ -139,6 +151,8 @@ class PRTable(object):
139 except sqlite3.IntegrityError as exc: 151 except sqlite3.IntegrityError as exc:
140 logger.error(str(exc)) 152 logger.error(str(exc))
141 153
154 self.dirty = True
155
142 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, 156 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
143 (version,pkgarch,checksum,value)) 157 (version,pkgarch,checksum,value))
144 row=data.fetchone() 158 row=data.fetchone()
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 1b08d59130..6ab10972e7 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -122,6 +122,10 @@ class PRServer(SimpleXMLRPCServer):
122 def work_forever(self,): 122 def work_forever(self,):
123 self.quit = False 123 self.quit = False
124 self.timeout = 0.5 124 self.timeout = 0.5
125 iter_count = 1
126 # With 60 iterations between syncs and a 0.5 second timeout between
127 # iterations, this will sync if dirty every ~30 seconds.
128 iterations_between_sync = 60
125 129
126 logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" % 130 logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
127 (self.dbfile, self.host, self.port, str(os.getpid()))) 131 (self.dbfile, self.host, self.port, str(os.getpid())))
@@ -129,6 +133,9 @@ class PRServer(SimpleXMLRPCServer):
129 self.handlerthread.start() 133 self.handlerthread.start()
130 while not self.quit: 134 while not self.quit:
131 self.handle_request() 135 self.handle_request()
136 iter_count = (iter_count + 1) % iterations_between_sync
137 if iter_count == 0:
138 self.table.sync_if_dirty()
132 139
133 self.table.sync() 140 self.table.sync()
134 logger.info("PRServer: stopping...") 141 logger.info("PRServer: stopping...")