diff options
Diffstat (limited to 'bitbake/lib/prserv/db.py')
-rw-r--r-- | bitbake/lib/prserv/db.py | 209 |
1 files changed, 177 insertions, 32 deletions
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py index bbee9316b2..f267daed13 100644 --- a/bitbake/lib/prserv/db.py +++ b/bitbake/lib/prserv/db.py | |||
@@ -1,9 +1,7 @@ | |||
1 | import logging | 1 | import logging |
2 | import os.path | 2 | import os.path |
3 | import errno | 3 | import errno |
4 | import sys | 4 | import prserv |
5 | import warnings | ||
6 | import sqlite3 | ||
7 | 5 | ||
8 | try: | 6 | try: |
9 | import sqlite3 | 7 | import sqlite3 |
@@ -14,73 +12,220 @@ sqlversion = sqlite3.sqlite_version_info | |||
14 | if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): | 12 | if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): |
15 | raise Exception("sqlite3 version 3.3.0 or later is required.") | 13 | raise Exception("sqlite3 version 3.3.0 or later is required.") |
16 | 14 | ||
17 | class NotFoundError(StandardError): | ||
18 | pass | ||
19 | |||
20 | class PRTable(): | 15 | class PRTable(): |
21 | def __init__(self,cursor,table): | 16 | def __init__(self, conn, table, nohist): |
22 | self.cursor = cursor | 17 | self.conn = conn |
23 | self.table = table | 18 | self.nohist = nohist |
19 | if nohist: | ||
20 | self.table = "%s_nohist" % table | ||
21 | else: | ||
22 | self.table = "%s_hist" % table | ||
24 | 23 | ||
25 | #create the table | ||
26 | self._execute("CREATE TABLE IF NOT EXISTS %s \ | 24 | self._execute("CREATE TABLE IF NOT EXISTS %s \ |
27 | (version TEXT NOT NULL, \ | 25 | (version TEXT NOT NULL, \ |
26 | pkgarch TEXT NOT NULL, \ | ||
28 | checksum TEXT NOT NULL, \ | 27 | checksum TEXT NOT NULL, \ |
29 | value INTEGER, \ | 28 | value INTEGER, \ |
30 | PRIMARY KEY (version,checksum));" | 29 | PRIMARY KEY (version, pkgarch, checksum));" % self.table) |
31 | % table) | ||
32 | 30 | ||
33 | def _execute(self, *query): | 31 | def _execute(self, *query): |
34 | """Execute a query, waiting to acquire a lock if necessary""" | 32 | """Execute a query, waiting to acquire a lock if necessary""" |
35 | count = 0 | 33 | count = 0 |
36 | while True: | 34 | while True: |
37 | try: | 35 | try: |
38 | return self.cursor.execute(*query) | 36 | return self.conn.execute(*query) |
39 | except sqlite3.OperationalError as exc: | 37 | except sqlite3.OperationalError as exc: |
40 | if 'database is locked' in str(exc) and count < 500: | 38 | if 'database is locked' in str(exc) and count < 500: |
41 | count = count + 1 | 39 | count = count + 1 |
42 | continue | 40 | continue |
43 | raise | 41 | raise exc |
44 | except sqlite3.IntegrityError as exc: | ||
45 | print "Integrity error %s" % str(exc) | ||
46 | break | ||
47 | 42 | ||
48 | def getValue(self, version, checksum): | 43 | def _getValueHist(self, version, pkgarch, checksum): |
49 | data=self._execute("SELECT value FROM %s WHERE version=? AND checksum=?;" % self.table, | 44 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, |
50 | (version,checksum)) | 45 | (version, pkgarch, checksum)) |
51 | row=data.fetchone() | 46 | row=data.fetchone() |
52 | if row != None: | 47 | if row != None: |
53 | return row[0] | 48 | return row[0] |
54 | else: | 49 | else: |
55 | #no value found, try to insert | 50 | #no value found, try to insert |
56 | self._execute("INSERT INTO %s VALUES (?, ?, (select ifnull(max(value)+1,0) from %s where version=?));" | 51 | try: |
52 | self._execute("BEGIN") | ||
53 | self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" | ||
57 | % (self.table,self.table), | 54 | % (self.table,self.table), |
58 | (version,checksum,version)) | 55 | (version,pkgarch, checksum,version, pkgarch)) |
59 | data=self._execute("SELECT value FROM %s WHERE version=? AND checksum=?;" % self.table, | 56 | self.conn.commit() |
60 | (version,checksum)) | 57 | except sqlite3.IntegrityError as exc: |
58 | logging.error(str(exc)) | ||
59 | |||
60 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
61 | (version, pkgarch, checksum)) | ||
62 | row=data.fetchone() | ||
63 | if row != None: | ||
64 | return row[0] | ||
65 | else: | ||
66 | raise prserv.NotFoundError | ||
67 | |||
68 | def _getValueNohist(self, version, pkgarch, checksum): | ||
69 | data=self._execute("SELECT value FROM %s \ | ||
70 | WHERE version=? AND pkgarch=? AND checksum=? AND \ | ||
71 | value >= (select max(value) from %s where version=? AND pkgarch=?);" | ||
72 | % (self.table, self.table), | ||
73 | (version, pkgarch, checksum, version, pkgarch)) | ||
74 | row=data.fetchone() | ||
75 | if row != None: | ||
76 | return row[0] | ||
77 | else: | ||
78 | #no value found, try to insert | ||
79 | try: | ||
80 | self._execute("BEGIN") | ||
81 | self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" | ||
82 | % (self.table,self.table), | ||
83 | (version, pkgarch, checksum, version, pkgarch)) | ||
84 | self.conn.commit() | ||
85 | except sqlite3.IntegrityError as exc: | ||
86 | logging.error(str(exc)) | ||
87 | self.conn.rollback() | ||
88 | |||
89 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
90 | (version, pkgarch, checksum)) | ||
61 | row=data.fetchone() | 91 | row=data.fetchone() |
62 | if row != None: | 92 | if row != None: |
63 | return row[0] | 93 | return row[0] |
64 | else: | 94 | else: |
65 | raise NotFoundError | 95 | raise prserv.NotFoundError |
96 | |||
97 | def getValue(self, version, pkgarch, checksum): | ||
98 | if self.nohist: | ||
99 | return self._getValueNohist(version, pkgarch, checksum) | ||
100 | else: | ||
101 | return self._getValueHist(version, pkgarch, checksum) | ||
102 | |||
103 | def _importHist(self, version, pkgarch, checksum, value): | ||
104 | val = None | ||
105 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
106 | (version, pkgarch, checksum)) | ||
107 | row = data.fetchone() | ||
108 | if row != None: | ||
109 | val=row[0] | ||
110 | else: | ||
111 | #no value found, try to insert | ||
112 | try: | ||
113 | self._execute("BEGIN") | ||
114 | self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, ?);" % (self.table), | ||
115 | (version, pkgarch, checksum, value)) | ||
116 | self.conn.commit() | ||
117 | except sqlite3.IntegrityError as exc: | ||
118 | logging.error(str(exc)) | ||
119 | |||
120 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
121 | (version, pkgarch, checksum)) | ||
122 | row = data.fetchone() | ||
123 | if row != None: | ||
124 | val = row[0] | ||
125 | return val | ||
126 | |||
127 | def _importNohist(self, version, pkgarch, checksum, value): | ||
128 | try: | ||
129 | #try to insert | ||
130 | self._execute("BEGIN") | ||
131 | self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, ?);" % (self.table), | ||
132 | (version, pkgarch, checksum,value)) | ||
133 | self.conn.commit() | ||
134 | except sqlite3.IntegrityError as exc: | ||
135 | #already have the record, try to update | ||
136 | try: | ||
137 | self._execute("BEGIN") | ||
138 | self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" | ||
139 | % (self.table), | ||
140 | (value,version,pkgarch,checksum,value)) | ||
141 | self.conn.commit() | ||
142 | except sqlite3.IntegrityError as exc: | ||
143 | logging.error(str(exc)) | ||
144 | |||
145 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, | ||
146 | (version,pkgarch,checksum,value)) | ||
147 | row=data.fetchone() | ||
148 | if row != None: | ||
149 | return row[0] | ||
150 | else: | ||
151 | return None | ||
152 | |||
153 | def importone(self, version, pkgarch, checksum, value): | ||
154 | if self.nohist: | ||
155 | return self._importNohist(version, pkgarch, checksum, value) | ||
156 | else: | ||
157 | return self._importHist(version, pkgarch, checksum, value) | ||
158 | |||
159 | def export(self, version, pkgarch, checksum, colinfo): | ||
160 | metainfo = {} | ||
161 | #column info | ||
162 | if colinfo: | ||
163 | metainfo['tbl_name'] = self.table | ||
164 | metainfo['core_ver'] = prserv.__version__ | ||
165 | metainfo['col_info'] = [] | ||
166 | data = self._execute("PRAGMA table_info(%s);" % self.table) | ||
167 | for row in data: | ||
168 | col = {} | ||
169 | col['name'] = row['name'] | ||
170 | col['type'] = row['type'] | ||
171 | col['notnull'] = row['notnull'] | ||
172 | col['dflt_value'] = row['dflt_value'] | ||
173 | col['pk'] = row['pk'] | ||
174 | metainfo['col_info'].append(col) | ||
175 | |||
176 | #data info | ||
177 | datainfo = [] | ||
178 | |||
179 | if self.nohist: | ||
180 | sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ | ||
181 | (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \ | ||
182 | WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) | ||
183 | else: | ||
184 | sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table | ||
185 | sqlarg = [] | ||
186 | where = "" | ||
187 | if version: | ||
188 | where += "AND T1.version=? " | ||
189 | sqlarg.append(str(version)) | ||
190 | if pkgarch: | ||
191 | where += "AND T1.pkgarch=? " | ||
192 | sqlarg.append(str(pkgarch)) | ||
193 | if checksum: | ||
194 | where += "AND T1.checksum=? " | ||
195 | sqlarg.append(str(checksum)) | ||
196 | |||
197 | sqlstmt += where + ";" | ||
198 | |||
199 | if len(sqlarg): | ||
200 | data = self._execute(sqlstmt, tuple(sqlarg)) | ||
201 | else: | ||
202 | data = self._execute(sqlstmt) | ||
203 | for row in data: | ||
204 | if row['version']: | ||
205 | col = {} | ||
206 | col['version'] = row['version'] | ||
207 | col['pkgarch'] = row['pkgarch'] | ||
208 | col['checksum'] = row['checksum'] | ||
209 | col['value'] = row['value'] | ||
210 | datainfo.append(col) | ||
211 | return (metainfo, datainfo) | ||
66 | 212 | ||
67 | class PRData(object): | 213 | class PRData(object): |
68 | """Object representing the PR database""" | 214 | """Object representing the PR database""" |
69 | def __init__(self, filename): | 215 | def __init__(self, filename, nohist=True): |
70 | self.filename=os.path.abspath(filename) | 216 | self.filename=os.path.abspath(filename) |
217 | self.nohist=nohist | ||
71 | #build directory hierarchy | 218 | #build directory hierarchy |
72 | try: | 219 | try: |
73 | os.makedirs(os.path.dirname(self.filename)) | 220 | os.makedirs(os.path.dirname(self.filename)) |
74 | except OSError as e: | 221 | except OSError as e: |
75 | if e.errno != errno.EEXIST: | 222 | if e.errno != errno.EEXIST: |
76 | raise e | 223 | raise e |
77 | self.connection=sqlite3.connect(self.filename, timeout=5, | 224 | self.connection=sqlite3.connect(self.filename, isolation_level="DEFERRED") |
78 | isolation_level=None) | 225 | self.connection.row_factory=sqlite3.Row |
79 | self.cursor=self.connection.cursor() | ||
80 | self._tables={} | 226 | self._tables={} |
81 | 227 | ||
82 | def __del__(self): | 228 | def __del__(self): |
83 | print "PRData: closing DB %s" % self.filename | ||
84 | self.connection.close() | 229 | self.connection.close() |
85 | 230 | ||
86 | def __getitem__(self,tblname): | 231 | def __getitem__(self,tblname): |
@@ -90,11 +235,11 @@ class PRData(object): | |||
90 | if tblname in self._tables: | 235 | if tblname in self._tables: |
91 | return self._tables[tblname] | 236 | return self._tables[tblname] |
92 | else: | 237 | else: |
93 | tableobj = self._tables[tblname] = PRTable(self.cursor, tblname) | 238 | tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist) |
94 | return tableobj | 239 | return tableobj |
95 | 240 | ||
96 | def __delitem__(self, tblname): | 241 | def __delitem__(self, tblname): |
97 | if tblname in self._tables: | 242 | if tblname in self._tables: |
98 | del self._tables[tblname] | 243 | del self._tables[tblname] |
99 | logging.info("drop table %s" % (tblname)) | 244 | logging.info("drop table %s" % (tblname)) |
100 | self.cursor.execute("DROP TABLE IF EXISTS %s;" % tblname) | 245 | self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) |