summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2018-12-07 15:59:28 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-08 17:17:37 +0000
commit66318c5831e91f571c167b9d33150523bd30ee95 (patch)
tree86603de1a543fe6658e70574f5f1bbdde5e55ab8 /bitbake/lib
parent888d76d2215565435ce7b0a890e2a7624e213874 (diff)
downloadpoky-66318c5831e91f571c167b9d33150523bd30ee95.tar.gz
bitbake: bitbake: persist_data: Retry database setup
The configuration of the sqlite database can timeout due to locking under heavy load and should be subject to the same retry logic as the other statements. [YOCTO #13069] (Bitbake rev: 5a2a95b0396e39662968690b3065d2f88167a71c) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/persist_data.py83
1 files changed, 48 insertions, 35 deletions
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
index 41fcf2a41c..4468facd18 100644
--- a/bitbake/lib/bb/persist_data.py
+++ b/bitbake/lib/bb/persist_data.py
@@ -42,24 +42,31 @@ logger = logging.getLogger("BitBake.PersistData")
42class SQLTable(collections.MutableMapping): 42class SQLTable(collections.MutableMapping):
43 class _Decorators(object): 43 class _Decorators(object):
44 @staticmethod 44 @staticmethod
45 def retry(f): 45 def retry(*, reconnect=True):
46 """ 46 """
47 Decorator that restarts a function if a database locked sqlite 47 Decorator that restarts a function if a database locked sqlite
48 exception occurs. 48 exception occurs. If reconnect is True, the database connection
49 will be closed and reopened each time a failure occurs
49 """ 50 """
50 def wrap_func(self, *args, **kwargs): 51 def retry_wrapper(f):
51 count = 0 52 def wrap_func(self, *args, **kwargs):
52 while True: 53 # Reconnect if necessary
53 try: 54 if self.connection is None and reconnect:
54 return f(self, *args, **kwargs) 55 self.reconnect()
55 except sqlite3.OperationalError as exc: 56
56 if 'is locked' in str(exc) and count < 500: 57 count = 0
57 count = count + 1 58 while True:
58 self.connection.close() 59 try:
59 self.connection = connect(self.cachefile) 60 return f(self, *args, **kwargs)
60 continue 61 except sqlite3.OperationalError as exc:
61 raise 62 if 'is locked' in str(exc) and count < 500:
62 return wrap_func 63 count = count + 1
64 if reconnect:
65 self.reconnect()
66 continue
67 raise
68 return wrap_func
69 return retry_wrapper
63 70
64 @staticmethod 71 @staticmethod
65 def transaction(f): 72 def transaction(f):
@@ -86,11 +93,28 @@ class SQLTable(collections.MutableMapping):
86 def __init__(self, cachefile, table): 93 def __init__(self, cachefile, table):
87 self.cachefile = cachefile 94 self.cachefile = cachefile
88 self.table = table 95 self.table = table
89 self.connection = connect(self.cachefile)
90 96
97 self.connection = None
91 self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table) 98 self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
92 99
93 @_Decorators.retry 100 @_Decorators.retry(reconnect=False)
101 @_Decorators.transaction
102 def _setup_database(self, cursor):
103 cursor.execute("pragma synchronous = off;")
104 # Enable WAL and keep the autocheckpoint length small (the default is
105 # usually 1000). Persistent caches are usually read-mostly, so keeping
106 # this short will keep readers running quickly
107 cursor.execute("pragma journal_mode = WAL;")
108 cursor.execute("pragma wal_autocheckpoint = 100;")
109
110 def reconnect(self):
111 if self.connection is not None:
112 self.connection.close()
113 self.connection = sqlite3.connect(self.cachefile, timeout=5)
114 self.connection.text_factory = str
115 self._setup_database()
116
117 @_Decorators.retry()
94 @_Decorators.transaction 118 @_Decorators.transaction
95 def _execute_single(self, cursor, *query): 119 def _execute_single(self, cursor, *query):
96 """ 120 """
@@ -99,7 +123,7 @@ class SQLTable(collections.MutableMapping):
99 """ 123 """
100 cursor.execute(*query) 124 cursor.execute(*query)
101 125
102 @_Decorators.retry 126 @_Decorators.retry()
103 def _row_iter(self, f, *query): 127 def _row_iter(self, f, *query):
104 """ 128 """
105 Helper function that returns a row iterator. Each time __next__ is 129 Helper function that returns a row iterator. Each time __next__ is
@@ -141,7 +165,7 @@ class SQLTable(collections.MutableMapping):
141 def __exit__(self, *excinfo): 165 def __exit__(self, *excinfo):
142 self.connection.__exit__(*excinfo) 166 self.connection.__exit__(*excinfo)
143 167
144 @_Decorators.retry 168 @_Decorators.retry()
145 @_Decorators.transaction 169 @_Decorators.transaction
146 def __getitem__(self, cursor, key): 170 def __getitem__(self, cursor, key):
147 cursor.execute("SELECT * from %s where key=?;" % self.table, [key]) 171 cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
@@ -150,14 +174,14 @@ class SQLTable(collections.MutableMapping):
150 return row[1] 174 return row[1]
151 raise KeyError(key) 175 raise KeyError(key)
152 176
153 @_Decorators.retry 177 @_Decorators.retry()
154 @_Decorators.transaction 178 @_Decorators.transaction
155 def __delitem__(self, cursor, key): 179 def __delitem__(self, cursor, key):
156 if key not in self: 180 if key not in self:
157 raise KeyError(key) 181 raise KeyError(key)
158 cursor.execute("DELETE from %s where key=?;" % self.table, [key]) 182 cursor.execute("DELETE from %s where key=?;" % self.table, [key])
159 183
160 @_Decorators.retry 184 @_Decorators.retry()
161 @_Decorators.transaction 185 @_Decorators.transaction
162 def __setitem__(self, cursor, key, value): 186 def __setitem__(self, cursor, key, value):
163 if not isinstance(key, str): 187 if not isinstance(key, str):
@@ -172,13 +196,13 @@ class SQLTable(collections.MutableMapping):
172 else: 196 else:
173 cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value]) 197 cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
174 198
175 @_Decorators.retry 199 @_Decorators.retry()
176 @_Decorators.transaction 200 @_Decorators.transaction
177 def __contains__(self, cursor, key): 201 def __contains__(self, cursor, key):
178 cursor.execute('SELECT * from %s where key=?;' % self.table, [key]) 202 cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
179 return cursor.fetchone() is not None 203 return cursor.fetchone() is not None
180 204
181 @_Decorators.retry 205 @_Decorators.retry()
182 @_Decorators.transaction 206 @_Decorators.transaction
183 def __len__(self, cursor): 207 def __len__(self, cursor):
184 cursor.execute("SELECT COUNT(key) FROM %s;" % self.table) 208 cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
@@ -213,7 +237,7 @@ class SQLTable(collections.MutableMapping):
213 return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" % 237 return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
214 self.table) 238 self.table)
215 239
216 @_Decorators.retry 240 @_Decorators.retry()
217 @_Decorators.transaction 241 @_Decorators.transaction
218 def clear(self, cursor): 242 def clear(self, cursor):
219 cursor.execute("DELETE FROM %s;" % self.table) 243 cursor.execute("DELETE FROM %s;" % self.table)
@@ -270,17 +294,6 @@ class PersistData(object):
270 """ 294 """
271 del self.data[domain][key] 295 del self.data[domain][key]
272 296
273def connect(database):
274 connection = sqlite3.connect(database, timeout=5)
275 connection.execute("pragma synchronous = off;")
276 # Enable WAL and keep the autocheckpoint length small (the default is
277 # usually 1000). Persistent caches are usually read-mostly, so keeping
278 # this short will keep readers running quickly
279 connection.execute("pragma journal_mode = WAL;")
280 connection.execute("pragma wal_autocheckpoint = 100;")
281 connection.text_factory = str
282 return connection
283
284def persist(domain, d): 297def persist(domain, d):
285 """Convenience factory for SQLTable objects based upon metadata""" 298 """Convenience factory for SQLTable objects based upon metadata"""
286 import bb.utils 299 import bb.utils