diff options
-rw-r--r-- | bitbake/lib/bb/persist_data.py | 188 |
1 files changed, 135 insertions, 53 deletions
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py index bef7018614..1a6319f949 100644 --- a/bitbake/lib/bb/persist_data.py +++ b/bitbake/lib/bb/persist_data.py | |||
@@ -29,6 +29,7 @@ import warnings | |||
29 | from bb.compat import total_ordering | 29 | from bb.compat import total_ordering |
30 | from collections import Mapping | 30 | from collections import Mapping |
31 | import sqlite3 | 31 | import sqlite3 |
32 | import contextlib | ||
32 | 33 | ||
33 | sqlversion = sqlite3.sqlite_version_info | 34 | sqlversion = sqlite3.sqlite_version_info |
34 | if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): | 35 | if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): |
@@ -45,75 +46,154 @@ if hasattr(sqlite3, 'enable_shared_cache'): | |||
45 | 46 | ||
46 | @total_ordering | 47 | @total_ordering |
47 | class SQLTable(collections.MutableMapping): | 48 | class SQLTable(collections.MutableMapping): |
49 | class _Decorators(object): | ||
50 | @staticmethod | ||
51 | def retry(f): | ||
52 | """ | ||
53 | Decorator that restarts a function if a database locked sqlite | ||
54 | exception occurs. | ||
55 | """ | ||
56 | def wrap_func(self, *args, **kwargs): | ||
57 | count = 0 | ||
58 | while True: | ||
59 | try: | ||
60 | return f(self, *args, **kwargs) | ||
61 | except sqlite3.OperationalError as exc: | ||
62 | if 'is locked' in str(exc) and count < 500: | ||
63 | count = count + 1 | ||
64 | self.connection.close() | ||
65 | self.connection = connect(self.cachefile) | ||
66 | continue | ||
67 | raise | ||
68 | return wrap_func | ||
69 | |||
70 | @staticmethod | ||
71 | def transaction(f): | ||
72 | """ | ||
73 | Decorator that starts a database transaction and creates a database | ||
74 | cursor for performing queries. If no exception is thrown, the | ||
75 | database results are commited. If an exception occurs, the database | ||
76 | is rolled back. In all cases, the cursor is closed after the | ||
77 | function ends. | ||
78 | |||
79 | Note that the cursor is passed as an extra argument to the function | ||
80 | after `self` and before any of the normal arguments | ||
81 | """ | ||
82 | def wrap_func(self, *args, **kwargs): | ||
83 | # Context manager will COMMIT the database on success, | ||
84 | # or ROLLBACK on an exception | ||
85 | with self.connection: | ||
86 | # Automatically close the cursor when done | ||
87 | with contextlib.closing(self.connection.cursor()) as cursor: | ||
88 | return f(self, cursor, *args, **kwargs) | ||
89 | return wrap_func | ||
90 | |||
48 | """Object representing a table/domain in the database""" | 91 | """Object representing a table/domain in the database""" |
49 | def __init__(self, cachefile, table): | 92 | def __init__(self, cachefile, table): |
50 | self.cachefile = cachefile | 93 | self.cachefile = cachefile |
51 | self.table = table | 94 | self.table = table |
52 | self.cursor = connect(self.cachefile) | 95 | self.connection = connect(self.cachefile) |
53 | 96 | ||
54 | self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" | 97 | self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % table) |
55 | % table) | 98 | |
56 | 99 | @_Decorators.retry | |
57 | def _execute(self, *query): | 100 | @_Decorators.transaction |
58 | """Execute a query, waiting to acquire a lock if necessary""" | 101 | def _execute_single(self, cursor, *query): |
59 | count = 0 | 102 | """ |
60 | while True: | 103 | Executes a single query and discards the results. This correctly closes |
61 | try: | 104 | the database cursor when finished |
62 | return self.cursor.execute(*query) | 105 | """ |
63 | except sqlite3.OperationalError as exc: | 106 | cursor.execute(*query) |
64 | if 'database is locked' in str(exc) and count < 500: | 107 | |
65 | count = count + 1 | 108 | @_Decorators.retry |
109 | def _row_iter(self, f, *query): | ||
110 | """ | ||
111 | Helper function that returns a row iterator. Each time __next__ is | ||
112 | called on the iterator, the provided function is evaluated to determine | ||
113 | the return value | ||
114 | """ | ||
115 | class CursorIter(object): | ||
116 | def __init__(self, cursor): | ||
117 | self.cursor = cursor | ||
118 | |||
119 | def __iter__(self): | ||
120 | return self | ||
121 | |||
122 | def __next__(self): | ||
123 | row = self.cursor.fetchone() | ||
124 | if row is None: | ||
66 | self.cursor.close() | 125 | self.cursor.close() |
67 | self.cursor = connect(self.cachefile) | 126 | raise StopIteration |
68 | continue | 127 | return f(row) |
69 | raise | 128 | |
129 | def __enter__(self): | ||
130 | return self | ||
131 | |||
132 | def __exit__(self, typ, value, traceback): | ||
133 | self.cursor.close() | ||
134 | return False | ||
135 | |||
136 | cursor = self.connection.cursor() | ||
137 | try: | ||
138 | cursor.execute(*query) | ||
139 | return CursorIter(cursor) | ||
140 | except: | ||
141 | cursor.close() | ||
70 | 142 | ||
71 | def __enter__(self): | 143 | def __enter__(self): |
72 | self.cursor.__enter__() | 144 | self.connection.__enter__() |
73 | return self | 145 | return self |
74 | 146 | ||
75 | def __exit__(self, *excinfo): | 147 | def __exit__(self, *excinfo): |
76 | self.cursor.__exit__(*excinfo) | 148 | self.connection.__exit__(*excinfo) |
77 | 149 | ||
78 | def __getitem__(self, key): | 150 | @_Decorators.retry |
79 | data = self._execute("SELECT * from %s where key=?;" % | 151 | @_Decorators.transaction |
80 | self.table, [key]) | 152 | def __getitem__(self, cursor, key): |
81 | for row in data: | 153 | cursor.execute("SELECT * from %s where key=?;" % self.table, [key]) |
154 | row = cursor.fetchone() | ||
155 | if row is not None: | ||
82 | return row[1] | 156 | return row[1] |
83 | raise KeyError(key) | 157 | raise KeyError(key) |
84 | 158 | ||
85 | def __delitem__(self, key): | 159 | @_Decorators.retry |
160 | @_Decorators.transaction | ||
161 | def __delitem__(self, cursor, key): | ||
86 | if key not in self: | 162 | if key not in self: |
87 | raise KeyError(key) | 163 | raise KeyError(key) |
88 | self._execute("DELETE from %s where key=?;" % self.table, [key]) | 164 | cursor.execute("DELETE from %s where key=?;" % self.table, [key]) |
89 | 165 | ||
90 | def __setitem__(self, key, value): | 166 | @_Decorators.retry |
167 | @_Decorators.transaction | ||
168 | def __setitem__(self, cursor, key, value): | ||
91 | if not isinstance(key, str): | 169 | if not isinstance(key, str): |
92 | raise TypeError('Only string keys are supported') | 170 | raise TypeError('Only string keys are supported') |
93 | elif not isinstance(value, str): | 171 | elif not isinstance(value, str): |
94 | raise TypeError('Only string values are supported') | 172 | raise TypeError('Only string values are supported') |
95 | 173 | ||
96 | data = self._execute("SELECT * from %s where key=?;" % | 174 | cursor.execute("SELECT * from %s where key=?;" % self.table, [key]) |
97 | self.table, [key]) | 175 | row = cursor.fetchone() |
98 | exists = len(list(data)) | 176 | if row is not None: |
99 | if exists: | 177 | cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key]) |
100 | self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table, | ||
101 | [value, key]) | ||
102 | else: | 178 | else: |
103 | self._execute("INSERT into %s(key, value) values (?, ?);" % | 179 | cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value]) |
104 | self.table, [key, value]) | 180 | |
105 | 181 | @_Decorators.retry | |
106 | def __contains__(self, key): | 182 | @_Decorators.transaction |
107 | return key in set(self) | 183 | def __contains__(self, cursor, key): |
108 | 184 | cursor.execute('SELECT * from %s where key=?;' % self.table, [key]) | |
109 | def __len__(self): | 185 | return cursor.fetchone() is not None |
110 | data = self._execute("SELECT COUNT(key) FROM %s;" % self.table) | 186 | |
111 | for row in data: | 187 | @_Decorators.retry |
188 | @_Decorators.transaction | ||
189 | def __len__(self, cursor): | ||
190 | cursor.execute("SELECT COUNT(key) FROM %s;" % self.table) | ||
191 | row = cursor.fetchone() | ||
192 | if row is not None: | ||
112 | return row[0] | 193 | return row[0] |
113 | 194 | ||
114 | def __iter__(self): | 195 | def __iter__(self): |
115 | data = self._execute("SELECT key FROM %s;" % self.table) | 196 | return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table) |
116 | return (row[0] for row in data) | ||
117 | 197 | ||
118 | def __lt__(self, other): | 198 | def __lt__(self, other): |
119 | if not isinstance(other, Mapping): | 199 | if not isinstance(other, Mapping): |
@@ -122,25 +202,27 @@ class SQLTable(collections.MutableMapping): | |||
122 | return len(self) < len(other) | 202 | return len(self) < len(other) |
123 | 203 | ||
124 | def get_by_pattern(self, pattern): | 204 | def get_by_pattern(self, pattern): |
125 | data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" % | 205 | return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" % |
126 | self.table, [pattern]) | 206 | self.table, [pattern]) |
127 | return [row[1] for row in data] | ||
128 | 207 | ||
129 | def values(self): | 208 | def values(self): |
130 | return list(self.itervalues()) | 209 | return list(self.itervalues()) |
131 | 210 | ||
132 | def itervalues(self): | 211 | def itervalues(self): |
133 | data = self._execute("SELECT value FROM %s;" % self.table) | 212 | return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" % |
134 | return (row[0] for row in data) | 213 | self.table) |
135 | 214 | ||
136 | def items(self): | 215 | def items(self): |
137 | return list(self.iteritems()) | 216 | return list(self.iteritems()) |
138 | 217 | ||
139 | def iteritems(self): | 218 | def iteritems(self): |
140 | return self._execute("SELECT * FROM %s;" % self.table) | 219 | return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" % |
220 | self.table) | ||
141 | 221 | ||
142 | def clear(self): | 222 | @_Decorators.retry |
143 | self._execute("DELETE FROM %s;" % self.table) | 223 | @_Decorators.transaction |
224 | def clear(self, cursor): | ||
225 | cursor.execute("DELETE FROM %s;" % self.table) | ||
144 | 226 | ||
145 | def has_key(self, key): | 227 | def has_key(self, key): |
146 | return key in self | 228 | return key in self |
@@ -195,7 +277,7 @@ class PersistData(object): | |||
195 | del self.data[domain][key] | 277 | del self.data[domain][key] |
196 | 278 | ||
197 | def connect(database): | 279 | def connect(database): |
198 | connection = sqlite3.connect(database, timeout=5, isolation_level=None) | 280 | connection = sqlite3.connect(database, timeout=5) |
199 | connection.execute("pragma synchronous = off;") | 281 | connection.execute("pragma synchronous = off;") |
200 | connection.text_factory = str | 282 | connection.text_factory = str |
201 | return connection | 283 | return connection |