From 295b75cf1cc17f9ab3990d9640a72af83a593a77 Mon Sep 17 00:00:00 2001 From: Paul Barker Date: Thu, 19 Aug 2021 12:46:44 -0400 Subject: bitbake: prserv: Add read-only mode [YOCTO #13659] (Bitbake rev: 44287430b9804fcbf2440f85a2424792140e4dc9) Signed-off-by: Paul Barker [updated for asyncrpc changes] Signed-off-by: Scott Murray Signed-off-by: Richard Purdie --- bitbake/lib/prserv/client.py | 9 +++++- bitbake/lib/prserv/db.py | 65 ++++++++++++++++++++++++++++++++++---------- bitbake/lib/prserv/serv.py | 40 ++++++++++++++++----------- 3 files changed, 83 insertions(+), 31 deletions(-) (limited to 'bitbake/lib/prserv') diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py index 285dce72f6..a3f19ddafc 100644 --- a/bitbake/lib/prserv/client.py +++ b/bitbake/lib/prserv/client.py @@ -32,10 +32,17 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): if response: return (response['metainfo'], response['datainfo']) + async def is_readonly(self): + response = await self.send_message( + {'is-readonly': {}} + ) + if response: + return response['readonly'] + class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods('getPR', 'importone', 'export') + self._add_methods('getPR', 'importone', 'export', 'is_readonly') def _get_async_client(self): return PRAsyncClient() diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py index cb2a2461e0..2710d4a225 100644 --- a/bitbake/lib/prserv/db.py +++ b/bitbake/lib/prserv/db.py @@ -30,21 +30,29 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): # class PRTable(object): - def __init__(self, conn, table, nohist): + def __init__(self, conn, table, nohist, read_only): self.conn = conn self.nohist = nohist + self.read_only = read_only self.dirty = False if nohist: self.table = "%s_nohist" % table else: self.table = "%s_hist" % table - self._execute("CREATE TABLE IF NOT EXISTS %s \ - (version TEXT NOT NULL, \ - pkgarch TEXT NOT NULL, \ - checksum TEXT NOT NULL, \ - value INTEGER, \ - PRIMARY KEY (version, pkgarch, checksum));" % self.table) + if self.read_only: + table_exists = self._execute( + "SELECT count(*) FROM sqlite_master \ + WHERE type='table' AND name='%s'" % (self.table)) + if not table_exists: + raise prserv.NotFoundError + else: + self._execute("CREATE TABLE IF NOT EXISTS %s \ + (version TEXT NOT NULL, \ + pkgarch TEXT NOT NULL, \ + checksum TEXT NOT NULL, \ + value INTEGER, \ + PRIMARY KEY (version, pkgarch, checksum));" % self.table) def _execute(self, *query): """Execute a query, waiting to acquire a lock if necessary""" @@ -59,8 +67,9 @@ class PRTable(object): raise exc def sync(self): - self.conn.commit() - self._execute("BEGIN EXCLUSIVE TRANSACTION") + if not self.read_only: + self.conn.commit() + self._execute("BEGIN EXCLUSIVE TRANSACTION") def sync_if_dirty(self): if self.dirty: @@ -75,6 +84,15 @@ class PRTable(object): return row[0] else: #no value found, try to insert + if self.read_only: + data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + (version, pkgarch)) + row = data.fetchone() + if row is not None: + return row[0] + else: + return 0 + try: self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" % (self.table,self.table), @@ -103,6 +121,15 @@ class PRTable(object): return row[0] else: #no value found, try to insert + if self.read_only: + data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + (version, pkgarch)) + row = data.fetchone() + if row is not None: + return row[0] + else: + return 0 + try: self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" % (self.table,self.table), @@ -128,6 +155,9 @@ class PRTable(object): return self._getValueHist(version, pkgarch, checksum) def _importHist(self, version, pkgarch, checksum, value): + if self.read_only: + return None + val = None data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) @@ -152,6 +182,9 @@ class PRTable(object): return val def _importNohist(self, version, pkgarch, checksum, value): + if self.read_only: + return None + try: #try to insert self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), @@ -245,19 +278,23 @@ class PRTable(object): class PRData(object): """Object representing the PR database""" - def __init__(self, filename, nohist=True): + def __init__(self, filename, nohist=True, read_only=False): self.filename=os.path.abspath(filename) self.nohist=nohist + self.read_only = read_only #build directory hierarchy try: os.makedirs(os.path.dirname(self.filename)) except OSError as e: if e.errno != errno.EEXIST: raise e - self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False) + uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") + logger.debug("Opening PRServ database '%s'" % (uri)) + self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) self.connection.row_factory=sqlite3.Row - self.connection.execute("pragma synchronous = off;") - self.connection.execute("PRAGMA journal_mode = MEMORY;") + if not self.read_only: + self.connection.execute("pragma synchronous = off;") + self.connection.execute("PRAGMA journal_mode = MEMORY;") self._tables={} def disconnect(self): @@ -270,7 +307,7 @@ class PRData(object): if tblname in self._tables: return self._tables[tblname] else: - tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist) + tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) return tableobj def __delitem__(self, tblname): diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py index 1fa4e1766c..17ae40967c 100644 --- a/bitbake/lib/prserv/serv.py +++ b/bitbake/lib/prserv/serv.py @@ -18,14 +18,16 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid" singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): - def __init__(self, reader, writer, table): + def __init__(self, reader, writer, table, read_only): super().__init__(reader, writer, 'PRSERVICE', logger) self.handlers.update({ 'get-pr': self.handle_get_pr, 'import-one': self.handle_import_one, 'export': self.handle_export, + 'is-readonly': self.handle_is_readonly, }) self.table = table + self.read_only = read_only def validate_proto_version(self): return (self.proto_version == (1, 0)) @@ -56,16 +58,17 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.write_message(response) async def handle_import_one(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - value = request['value'] + response = None + if not self.read_only: + version = request['version'] + pkgarch = request['pkgarch'] + checksum = request['checksum'] + value = request['value'] + + value = self.table.importone(version, pkgarch, checksum, value) + if value is not None: + response = {'value': value} - value = self.table.importone(version, pkgarch, checksum, value) - if value is not None: - response = {'value': value} - else: - response = None self.write_message(response) async def handle_export(self, request): @@ -83,20 +86,25 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = {'metainfo': metainfo, 'datainfo': datainfo} self.write_message(response) + async def handle_is_readonly(self, request): + response = {'readonly': self.read_only} + self.write_message(response) + class PRServer(bb.asyncrpc.AsyncServer): - def __init__(self, dbfile): + def __init__(self, dbfile, read_only=False): super().__init__(logger) self.dbfile = dbfile self.table = None + self.read_only = read_only def accept_client(self, reader, writer): - return PRServerClient(reader, writer, self.table) + return PRServerClient(reader, writer, self.table, self.read_only) def _serve_forever(self): - self.db = prserv.db.PRData(self.dbfile) + self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only) self.table = self.db["PRMAIN"] - logger.debug("Started PRServer with DBfile: %s, Address: %s, PID: %s" % + logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) super()._serve_forever() @@ -194,7 +202,7 @@ def run_as_daemon(func, pidfile, logfile): os.remove(pidfile) os._exit(0) -def start_daemon(dbfile, host, port, logfile): +def start_daemon(dbfile, host, port, logfile, read_only=False): ip = socket.gethostbyname(host) pidfile = PIDPREFIX % (ip, port) try: @@ -210,7 +218,7 @@ def start_daemon(dbfile, host, port, logfile): dbfile = os.path.abspath(dbfile) def daemon_main(): - server = PRServer(dbfile) + server = PRServer(dbfile, read_only=read_only) server.start_tcp_server(host, port) server.serve_forever() -- cgit v1.2.3-54-g00ecf