summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/prserv/db.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/prserv/db.py')
-rw-r--r--bitbake/lib/prserv/db.py452
1 files changed, 196 insertions, 256 deletions
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index eb41508198..2da493ddf5 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -8,19 +8,13 @@ import logging
8import os.path 8import os.path
9import errno 9import errno
10import prserv 10import prserv
11import time 11import sqlite3
12 12
13try: 13from contextlib import closing
14 import sqlite3 14from . import increase_revision, revision_greater, revision_smaller
15except ImportError:
16 from pysqlite2 import dbapi2 as sqlite3
17 15
18logger = logging.getLogger("BitBake.PRserv") 16logger = logging.getLogger("BitBake.PRserv")
19 17
20sqlversion = sqlite3.sqlite_version_info
21if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
22 raise Exception("sqlite3 version 3.3.0 or later is required.")
23
24# 18#
25# "No History" mode - for a given query tuple (version, pkgarch, checksum), 19# "No History" mode - for a given query tuple (version, pkgarch, checksum),
26# the returned value will be the largest among all the values of the same 20# the returned value will be the largest among all the values of the same
@@ -29,287 +23,232 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
29# "History" mode - Return a new higher value for previously unseen query 23# "History" mode - Return a new higher value for previously unseen query
30# tuple (version, pkgarch, checksum), otherwise return historical value. 24# tuple (version, pkgarch, checksum), otherwise return historical value.
31# Value can decrement if returning to a previous build. 25# Value can decrement if returning to a previous build.
32#
33 26
34class PRTable(object): 27class PRTable(object):
35 def __init__(self, conn, table, nohist, read_only): 28 def __init__(self, conn, table, read_only):
36 self.conn = conn 29 self.conn = conn
37 self.nohist = nohist
38 self.read_only = read_only 30 self.read_only = read_only
39 self.dirty = False 31 self.table = table
40 if nohist: 32
41 self.table = "%s_nohist" % table 33 # Creating the table even if the server is read-only.
42 else: 34 # This avoids a race condition if a shared database
43 self.table = "%s_hist" % table 35 # is accessed by a read-only server first.
44 36
45 if self.read_only: 37 with closing(self.conn.cursor()) as cursor:
46 table_exists = self._execute( 38 cursor.execute("CREATE TABLE IF NOT EXISTS %s \
47 "SELECT count(*) FROM sqlite_master \
48 WHERE type='table' AND name='%s'" % (self.table))
49 if not table_exists:
50 raise prserv.NotFoundError
51 else:
52 self._execute("CREATE TABLE IF NOT EXISTS %s \
53 (version TEXT NOT NULL, \ 39 (version TEXT NOT NULL, \
54 pkgarch TEXT NOT NULL, \ 40 pkgarch TEXT NOT NULL, \
55 checksum TEXT NOT NULL, \ 41 checksum TEXT NOT NULL, \
56 value INTEGER, \ 42 value TEXT, \
57 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 43 PRIMARY KEY (version, pkgarch, checksum, value));" % self.table)
58
59 def _execute(self, *query):
60 """Execute a query, waiting to acquire a lock if necessary"""
61 start = time.time()
62 end = start + 20
63 while True:
64 try:
65 return self.conn.execute(*query)
66 except sqlite3.OperationalError as exc:
67 if "is locked" in str(exc) and end > time.time():
68 continue
69 raise exc
70
71 def sync(self):
72 if not self.read_only:
73 self.conn.commit() 44 self.conn.commit()
74 self._execute("BEGIN EXCLUSIVE TRANSACTION")
75 45
76 def sync_if_dirty(self): 46 def _extremum_value(self, rows, is_max):
77 if self.dirty: 47 value = None
78 self.sync() 48
79 self.dirty = False 49 for row in rows:
50 current_value = row[0]
51 if value is None:
52 value = current_value
53 else:
54 if is_max:
55 is_new_extremum = revision_greater(current_value, value)
56 else:
57 is_new_extremum = revision_smaller(current_value, value)
58 if is_new_extremum:
59 value = current_value
60 return value
61
62 def _max_value(self, rows):
63 return self._extremum_value(rows, True)
64
65 def _min_value(self, rows):
66 return self._extremum_value(rows, False)
80 67
81 def test_package(self, version, pkgarch): 68 def test_package(self, version, pkgarch):
82 """Returns whether the specified package version is found in the database for the specified architecture""" 69 """Returns whether the specified package version is found in the database for the specified architecture"""
83 70
84 # Just returns the value if found or None otherwise 71 # Just returns the value if found or None otherwise
85 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, 72 with closing(self.conn.cursor()) as cursor:
86 (version, pkgarch)) 73 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
87 row=data.fetchone() 74 (version, pkgarch))
88 if row is not None: 75 row=data.fetchone()
89 return True 76 if row is not None:
90 else: 77 return True
91 return False 78 else:
79 return False
80
81 def test_checksum_value(self, version, pkgarch, checksum, value):
82 """Returns whether the specified value is found in the database for the specified package, architecture and checksum"""
83
84 with closing(self.conn.cursor()) as cursor:
85 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and checksum=? and value=?;" % self.table,
86 (version, pkgarch, checksum, value))
87 row=data.fetchone()
88 if row is not None:
89 return True
90 else:
91 return False
92 92
93 def test_value(self, version, pkgarch, value): 93 def test_value(self, version, pkgarch, value):
94 """Returns whether the specified value is found in the database for the specified package and architecture""" 94 """Returns whether the specified value is found in the database for the specified package and architecture"""
95 95
96 # Just returns the value if found or None otherwise 96 # Just returns the value if found or None otherwise
97 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, 97 with closing(self.conn.cursor()) as cursor:
98 (version, pkgarch, value)) 98 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
99 row=data.fetchone() 99 (version, pkgarch, value))
100 if row is not None: 100 row=data.fetchone()
101 return True 101 if row is not None:
102 else: 102 return True
103 return False 103 else:
104 return False
104 105
105 def find_value(self, version, pkgarch, checksum): 106
107 def find_package_max_value(self, version, pkgarch):
108 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
109
110 with closing(self.conn.cursor()) as cursor:
111 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table),
112 (version, pkgarch))
113 rows = data.fetchall()
114 value = self._max_value(rows)
115 return value
116
117 def find_value(self, version, pkgarch, checksum, history=False):
106 """Returns the value for the specified checksum if found or None otherwise.""" 118 """Returns the value for the specified checksum if found or None otherwise."""
107 119
108 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 120 if history:
109 (version, pkgarch, checksum)) 121 return self.find_min_value(version, pkgarch, checksum)
110 row=data.fetchone()
111 if row is not None:
112 return row[0]
113 else: 122 else:
114 return None 123 return self.find_max_value(version, pkgarch, checksum)
115 124
116 def find_max_value(self, version, pkgarch):
117 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
118 125
119 data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), 126 def _find_extremum_value(self, version, pkgarch, checksum, is_max):
120 (version, pkgarch)) 127 """Returns the maximum (if is_max is True) or minimum (if is_max is False) value
121 row = data.fetchone() 128 for (version, pkgarch, checksum), or None if not found. Doesn't create a new value"""
122 if row is not None:
123 return row[0]
124 else:
125 return None
126
127 def _get_value_hist(self, version, pkgarch, checksum):
128 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
129 (version, pkgarch, checksum))
130 row=data.fetchone()
131 if row is not None:
132 return row[0]
133 else:
134 #no value found, try to insert
135 if self.read_only:
136 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
137 (version, pkgarch))
138 row = data.fetchone()
139 if row is not None:
140 return row[0]
141 else:
142 return 0
143 129
144 try: 130 with closing(self.conn.cursor()) as cursor:
145 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" 131 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table),
146 % (self.table, self.table), 132 (version, pkgarch, checksum))
147 (version, pkgarch, checksum, version, pkgarch)) 133 rows = data.fetchall()
148 except sqlite3.IntegrityError as exc: 134 return self._extremum_value(rows, is_max)
149 logger.error(str(exc))
150 135
151 self.dirty = True 136 def find_max_value(self, version, pkgarch, checksum):
137 return self._find_extremum_value(version, pkgarch, checksum, True)
152 138
153 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 139 def find_min_value(self, version, pkgarch, checksum):
154 (version, pkgarch, checksum)) 140 return self._find_extremum_value(version, pkgarch, checksum, False)
155 row=data.fetchone() 141
156 if row is not None: 142 def find_new_subvalue(self, version, pkgarch, base):
157 return row[0] 143 """Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found.
158 else: 144 This doesn't store a new value."""
159 raise prserv.NotFoundError 145
160 146 with closing(self.conn.cursor()) as cursor:
161 def _get_value_no_hist(self, version, pkgarch, checksum): 147 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base),
162 data=self._execute("SELECT value FROM %s \ 148 (version, pkgarch))
163 WHERE version=? AND pkgarch=? AND checksum=? AND \ 149 rows = data.fetchall()
164 value >= (select max(value) from %s where version=? AND pkgarch=?);" 150 value = self._max_value(rows)
165 % (self.table, self.table), 151
166 (version, pkgarch, checksum, version, pkgarch)) 152 if value is not None:
167 row=data.fetchone() 153 return increase_revision(value)
168 if row is not None:
169 return row[0]
170 else:
171 #no value found, try to insert
172 if self.read_only:
173 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
174 (version, pkgarch))
175 return data.fetchone()[0]
176
177 try:
178 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
179 % (self.table, self.table),
180 (version, pkgarch, checksum, version, pkgarch))
181 except sqlite3.IntegrityError as exc:
182 logger.error(str(exc))
183 self.conn.rollback()
184
185 self.dirty = True
186
187 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
188 (version, pkgarch, checksum))
189 row=data.fetchone()
190 if row is not None:
191 return row[0]
192 else: 154 else:
193 raise prserv.NotFoundError 155 return base + ".0"
194 156
195 def get_value(self, version, pkgarch, checksum): 157 def store_value(self, version, pkgarch, checksum, value):
196 if self.nohist: 158 """Store value in the database"""
197 return self._get_value_no_hist(version, pkgarch, checksum) 159
198 else: 160 if not self.read_only and not self.test_checksum_value(version, pkgarch, checksum, value):
199 return self._get_value_hist(version, pkgarch, checksum) 161 with closing(self.conn.cursor()) as cursor:
200 162 cursor.execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
201 def _import_hist(self, version, pkgarch, checksum, value):
202 if self.read_only:
203 return None
204
205 val = None
206 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
207 (version, pkgarch, checksum))
208 row = data.fetchone()
209 if row is not None:
210 val=row[0]
211 else:
212 #no value found, try to insert
213 try:
214 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
215 (version, pkgarch, checksum, value)) 163 (version, pkgarch, checksum, value))
216 except sqlite3.IntegrityError as exc: 164 self.conn.commit()
217 logger.error(str(exc))
218 165
219 self.dirty = True 166 def _get_value(self, version, pkgarch, checksum, history):
220 167
221 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 168 max_value = self.find_package_max_value(version, pkgarch)
222 (version, pkgarch, checksum))
223 row = data.fetchone()
224 if row is not None:
225 val = row[0]
226 return val
227 169
228 def _import_no_hist(self, version, pkgarch, checksum, value): 170 if max_value is None:
229 if self.read_only: 171 # version, pkgarch completely unknown. Return initial value.
230 return None 172 return "0"
231 173
232 try: 174 value = self.find_value(version, pkgarch, checksum, history)
233 #try to insert 175
234 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 176 if value is None:
235 (version, pkgarch, checksum, value)) 177 # version, pkgarch found but not checksum. Create a new value from the maximum one
236 except sqlite3.IntegrityError as exc: 178 return increase_revision(max_value)
237 #already have the record, try to update 179
238 try: 180 if history:
239 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 181 return value
240 % (self.table), 182
241 (value, version, pkgarch, checksum, value)) 183 # "no history" mode - If the value is not the maximum value for the package, need to increase it.
242 except sqlite3.IntegrityError as exc: 184 if max_value > value:
243 logger.error(str(exc)) 185 return increase_revision(max_value)
244
245 self.dirty = True
246
247 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
248 (version, pkgarch, checksum, value))
249 row=data.fetchone()
250 if row is not None:
251 return row[0]
252 else: 186 else:
253 return None 187 return value
188
189 def get_value(self, version, pkgarch, checksum, history):
190 value = self._get_value(version, pkgarch, checksum, history)
191 if not self.read_only:
192 self.store_value(version, pkgarch, checksum, value)
193 return value
254 194
255 def importone(self, version, pkgarch, checksum, value): 195 def importone(self, version, pkgarch, checksum, value):
256 if self.nohist: 196 self.store_value(version, pkgarch, checksum, value)
257 return self._import_no_hist(version, pkgarch, checksum, value) 197 return value
258 else:
259 return self._import_hist(version, pkgarch, checksum, value)
260 198
261 def export(self, version, pkgarch, checksum, colinfo): 199 def export(self, version, pkgarch, checksum, colinfo, history=False):
262 metainfo = {} 200 metainfo = {}
263 #column info 201 with closing(self.conn.cursor()) as cursor:
264 if colinfo: 202 #column info
265 metainfo["tbl_name"] = self.table 203 if colinfo:
266 metainfo["core_ver"] = prserv.__version__ 204 metainfo["tbl_name"] = self.table
267 metainfo["col_info"] = [] 205 metainfo["core_ver"] = prserv.__version__
268 data = self._execute("PRAGMA table_info(%s);" % self.table) 206 metainfo["col_info"] = []
207 data = cursor.execute("PRAGMA table_info(%s);" % self.table)
208 for row in data:
209 col = {}
210 col["name"] = row["name"]
211 col["type"] = row["type"]
212 col["notnull"] = row["notnull"]
213 col["dflt_value"] = row["dflt_value"]
214 col["pk"] = row["pk"]
215 metainfo["col_info"].append(col)
216
217 #data info
218 datainfo = []
219
220 if history:
221 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
222 else:
223 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
224 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
225 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
226 sqlarg = []
227 where = ""
228 if version:
229 where += "AND T1.version=? "
230 sqlarg.append(str(version))
231 if pkgarch:
232 where += "AND T1.pkgarch=? "
233 sqlarg.append(str(pkgarch))
234 if checksum:
235 where += "AND T1.checksum=? "
236 sqlarg.append(str(checksum))
237
238 sqlstmt += where + ";"
239
240 if len(sqlarg):
241 data = cursor.execute(sqlstmt, tuple(sqlarg))
242 else:
243 data = cursor.execute(sqlstmt)
269 for row in data: 244 for row in data:
270 col = {} 245 if row["version"]:
271 col["name"] = row["name"] 246 col = {}
272 col["type"] = row["type"] 247 col["version"] = row["version"]
273 col["notnull"] = row["notnull"] 248 col["pkgarch"] = row["pkgarch"]
274 col["dflt_value"] = row["dflt_value"] 249 col["checksum"] = row["checksum"]
275 col["pk"] = row["pk"] 250 col["value"] = row["value"]
276 metainfo["col_info"].append(col) 251 datainfo.append(col)
277
278 #data info
279 datainfo = []
280
281 if self.nohist:
282 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
283 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
284 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
285 else:
286 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
287 sqlarg = []
288 where = ""
289 if version:
290 where += "AND T1.version=? "
291 sqlarg.append(str(version))
292 if pkgarch:
293 where += "AND T1.pkgarch=? "
294 sqlarg.append(str(pkgarch))
295 if checksum:
296 where += "AND T1.checksum=? "
297 sqlarg.append(str(checksum))
298
299 sqlstmt += where + ";"
300
301 if len(sqlarg):
302 data = self._execute(sqlstmt, tuple(sqlarg))
303 else:
304 data = self._execute(sqlstmt)
305 for row in data:
306 if row["version"]:
307 col = {}
308 col["version"] = row["version"]
309 col["pkgarch"] = row["pkgarch"]
310 col["checksum"] = row["checksum"]
311 col["value"] = row["value"]
312 datainfo.append(col)
313 return (metainfo, datainfo) 252 return (metainfo, datainfo)
314 253
315 def dump_db(self, fd): 254 def dump_db(self, fd):
@@ -322,9 +261,8 @@ class PRTable(object):
322 261
323class PRData(object): 262class PRData(object):
324 """Object representing the PR database""" 263 """Object representing the PR database"""
325 def __init__(self, filename, nohist=True, read_only=False): 264 def __init__(self, filename, read_only=False):
326 self.filename=os.path.abspath(filename) 265 self.filename=os.path.abspath(filename)
327 self.nohist=nohist
328 self.read_only = read_only 266 self.read_only = read_only
329 #build directory hierarchy 267 #build directory hierarchy
330 try: 268 try:
@@ -334,14 +272,15 @@ class PRData(object):
334 raise e 272 raise e
335 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 273 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "")
336 logger.debug("Opening PRServ database '%s'" % (uri)) 274 logger.debug("Opening PRServ database '%s'" % (uri))
337 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 275 self.connection=sqlite3.connect(uri, uri=True)
338 self.connection.row_factory=sqlite3.Row 276 self.connection.row_factory=sqlite3.Row
339 if not self.read_only: 277 self.connection.execute("PRAGMA synchronous = OFF;")
340 self.connection.execute("pragma synchronous = off;") 278 self.connection.execute("PRAGMA journal_mode = WAL;")
341 self.connection.execute("PRAGMA journal_mode = MEMORY;") 279 self.connection.commit()
342 self._tables={} 280 self._tables={}
343 281
344 def disconnect(self): 282 def disconnect(self):
283 self.connection.commit()
345 self.connection.close() 284 self.connection.close()
346 285
347 def __getitem__(self, tblname): 286 def __getitem__(self, tblname):
@@ -351,7 +290,7 @@ class PRData(object):
351 if tblname in self._tables: 290 if tblname in self._tables:
352 return self._tables[tblname] 291 return self._tables[tblname]
353 else: 292 else:
354 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 293 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only)
355 return tableobj 294 return tableobj
356 295
357 def __delitem__(self, tblname): 296 def __delitem__(self, tblname):
@@ -359,3 +298,4 @@ class PRData(object):
359 del self._tables[tblname] 298 del self._tables[tblname]
360 logger.info("drop table %s" % (tblname)) 299 logger.info("drop table %s" % (tblname))
361 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 300 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
301 self.connection.commit()