summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChris Larson <chris_larson@mentor.com>2010-09-21 18:11:54 -0700
committerRichard Purdie <rpurdie@linux.intel.com>2011-01-05 00:58:23 +0000
commit2671bb4197a06403e1cd3cedb0f6452046660d0a (patch)
tree317cba67a6a6d686598d7cb1aec735802f3e04da
parent3069c0b2588c9e88a4fa2fd4d37356410d364410 (diff)
downloadpoky-2671bb4197a06403e1cd3cedb0f6452046660d0a.tar.gz
Rework the persist_data API
Rather than having to run .addDomain() and then .getValue(domain, key), .setValue(domain, key), etc, now it just works as mappings. As an example: setValue(domain, key) -> persist[domain][key] = value It also arranges things so we can construct objects of this type using any arbitrary filename/path for the sqlite3 database, rather than being so tightly bound to the metadata. (Bitbake rev: d9e8b8af308ae871efdc8ef0782be30af8c1f894) Signed-off-by: Chris Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rw-r--r--bitbake/lib/bb/persist_data.py209
1 files changed, 137 insertions, 72 deletions
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
index 9558e71283..4f87c37f2b 100644
--- a/bitbake/lib/bb/persist_data.py
+++ b/bitbake/lib/bb/persist_data.py
@@ -1,6 +1,12 @@
1# BitBake Persistent Data Store 1"""BitBake Persistent Data Store
2# 2
3Used to store data in a central location such that other threads/tasks can
4access them at some future date. Acts as a convenience wrapper around sqlite,
5currently, providing a key/value store accessed by 'domain'.
6"""
7
3# Copyright (C) 2007 Richard Purdie 8# Copyright (C) 2007 Richard Purdie
9# Copyright (C) 2010 Chris Larson <chris_larson@mentor.com>
4# 10#
5# This program is free software; you can redistribute it and/or modify 11# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as 12# it under the terms of the GNU General Public License version 2 as
@@ -15,116 +21,175 @@
15# with this program; if not, write to the Free Software Foundation, Inc., 21# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 23
18import os 24import collections
19import logging 25import logging
20import bb 26import os.path
21import bb.utils 27import sys
22 28import warnings
23logger = logging.getLogger("BitBake.PersistData") 29import bb.msg, bb.data, bb.utils
24 30
25try: 31try:
26 import sqlite3 32 import sqlite3
27except ImportError: 33except ImportError:
28 try: 34 from pysqlite2 import dbapi2 as sqlite3
29 from pysqlite2 import dbapi2 as sqlite3
30 except ImportError:
31 bb.msg.fatal(bb.msg.domain.PersistData, "Importing sqlite3 and pysqlite2 failed, please install one of them. Python 2.5 or a 'python-pysqlite2' like package is likely to be what you need.")
32 35
33sqlversion = sqlite3.sqlite_version_info 36sqlversion = sqlite3.sqlite_version_info
34if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): 37if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
35 bb.msg.fatal(bb.msg.domain.PersistData, "sqlite3 version 3.3.0 or later is required.") 38 raise Exception("sqlite3 version 3.3.0 or later is required.")
36 39
37class PersistData:
38 """
39 BitBake Persistent Data Store
40 40
41 Used to store data in a central location such that other threads/tasks can 41logger = logging.getLogger("BitBake.PersistData")
42 access them at some future date.
43 42
44 The "domain" is used as a key to isolate each data pool and in this
45 implementation corresponds to an SQL table. The SQL table consists of a
46 simple key and value pair.
47 43
48 Why sqlite? It handles all the locking issues for us. 44class SQLTable(collections.MutableMapping):
49 """ 45 """Object representing a table/domain in the database"""
50 def __init__(self, d): 46 def __init__(self, cursor, table):
51 self.cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True) 47 self.cursor = cursor
52 if self.cachedir in [None, '']: 48 self.table = table
53 bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.") 49
54 try: 50 cursor.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
55 os.stat(self.cachedir) 51 % table)
56 except OSError: 52
57 bb.utils.mkdirhier(self.cachedir) 53 def _execute(self, *query):
54 """Execute a query, waiting to acquire a lock if necessary"""
55 count = 0
56 while True:
57 try:
58 return self.cursor.execute(*query)
59 break
60 except sqlite3.OperationalError as exc:
61 if 'database is locked' in str(exc) and count < 500:
62 count = count + 1
63 continue
64 raise
65
66 def __getitem__(self, key):
67 data = self.cursor.execute("SELECT * from %s where key=?;" %
68 self.table, [key])
69 for row in data:
70 return row[1]
71
72 def __delitem__(self, key):
73 self._execute("DELETE from %s where key=?;" % self.table, [key])
74
75 def __setitem__(self, key, value):
76 data = self.cursor.execute("SELECT * from %s where key=?;" %
77 self.table, [key])
78 exists = len(list(data))
79 if exists:
80 self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
81 [value, key])
82 else:
83 self._execute("INSERT into %s(key, value) values (?, ?);" %
84 self.table, [key, value])
85
86 def __contains__(self, key):
87 return key in set(self)
88
89 def __len__(self):
90 data = self.cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
91 for row in data:
92 return row[0]
93
94 def __iter__(self):
95 data = self.cursor.execute("SELECT key FROM %s;" % self.table)
96 for row in data:
97 yield row[0]
98
99 def iteritems(self):
100 data = self.cursor.execute("SELECT * FROM %s;" % self.table)
101 for row in data:
102 yield row[0], row[1]
103
104 def itervalues(self):
105 data = self.cursor.execute("SELECT value FROM %s;" % self.table)
106 for row in data:
107 yield row[0]
108
109
110class SQLData(object):
111 """Object representing the persistent data"""
112 def __init__(self, filename):
113 bb.utils.mkdirhier(os.path.dirname(filename))
114
115 self.filename = filename
116 self.connection = sqlite3.connect(filename, timeout=5,
117 isolation_level=None)
118 self.cursor = self.connection.cursor()
119 self._tables = {}
120
121 def __getitem__(self, table):
122 if not isinstance(table, basestring):
123 raise TypeError("table argument must be a string, not '%s'" %
124 type(table))
58 125
59 self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3") 126 if table in self._tables:
60 logger.debug(1, "Using '%s' as the persistent data cache", self.cachefile) 127 return self._tables[table]
128 else:
129 tableobj = self._tables[table] = SQLTable(self.cursor, table)
130 return tableobj
131
132 def __delitem__(self, table):
133 if table in self._tables:
134 del self._tables[table]
135 self.cursor.execute("DROP TABLE IF EXISTS %s;" % table)
61 136
62 self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None) 137
138class PersistData(object):
139 """Deprecated representation of the bitbake persistent data store"""
140 def __init__(self, d):
141 warnings.warn("Use of PersistData will be deprecated in the future",
142 category=PendingDeprecationWarning,
143 stacklevel=2)
144
145 self.data = persist(d)
146 logger.debug(1, "Using '%s' as the persistent data cache",
147 self.data.filename)
63 148
64 def addDomain(self, domain): 149 def addDomain(self, domain):
65 """ 150 """
66 Should be called before any domain is used 151 Add a domain (pending deprecation)
67 Creates it if it doesn't exist.
68 """ 152 """
69 self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain) 153 return self.data[domain]
70 154
71 def delDomain(self, domain): 155 def delDomain(self, domain):
72 """ 156 """
73 Removes a domain and all the data it contains 157 Removes a domain and all the data it contains
74 """ 158 """
75 self._execute("DROP TABLE IF EXISTS %s;" % domain) 159 del self.data[domain]
76 160
77 def getKeyValues(self, domain): 161 def getKeyValues(self, domain):
78 """ 162 """
79 Return a list of key + value pairs for a domain 163 Return a list of key + value pairs for a domain
80 """ 164 """
81 ret = {} 165 return self.data[domain].items()
82 data = self._execute("SELECT key, value from %s;" % domain)
83 for row in data:
84 ret[str(row[0])] = str(row[1])
85
86 return ret
87 166
88 def getValue(self, domain, key): 167 def getValue(self, domain, key):
89 """ 168 """
90 Return the value of a key for a domain 169 Return the value of a key for a domain
91 """ 170 """
92 data = self._execute("SELECT * from %s where key=?;" % domain, [key]) 171 return self.data[domain][key]
93 for row in data:
94 return row[1]
95 172
96 def setValue(self, domain, key, value): 173 def setValue(self, domain, key, value):
97 """ 174 """
98 Sets the value of a key for a domain 175 Sets the value of a key for a domain
99 """ 176 """
100 data = self._execute("SELECT * from %s where key=?;" % domain, [key]) 177 self.data[domain][key] = value
101 rows = 0
102 for row in data:
103 rows = rows + 1
104 if rows:
105 self._execute("UPDATE %s SET value=? WHERE key=?;" % domain, [value, key])
106 else:
107 self._execute("INSERT into %s(key, value) values (?, ?);" % domain, [key, value])
108 178
109 def delValue(self, domain, key): 179 def delValue(self, domain, key):
110 """ 180 """
111 Deletes a key/value pair 181 Deletes a key/value pair
112 """ 182 """
113 self._execute("DELETE from %s where key=?;" % domain, [key]) 183 del self.data[domain][key]
114 184
115 # 185
116 # We wrap the sqlite execute calls as on contended machines or single threaded 186def persist(d):
117 # systems we can have multiple processes trying to access the DB at once and it seems 187 """Convenience factory for construction of SQLData based upon metadata"""
118 # sqlite sometimes doesn't wait for the timeout. We therefore loop but put in an 188 cachedir = (bb.data.getVar("PERSISTENT_DIR", d, True) or
119 # emergency brake too 189 bb.data.getVar("CACHE", d, True))
120 # 190 if not cachedir:
121 def _execute(self, *query): 191 logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
122 count = 0 192 sys.exit(1)
123 while True: 193
124 try: 194 cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
125 return self.connection.execute(*query) 195 return SQLData(cachefile)
126 except sqlite3.OperationalError as e:
127 if 'database is locked' in str(e) and count < 500:
128 count = count + 1
129 continue
130 raise