"""BitBake Persistent Data Store Used to store data in a central location such that other threads/tasks can access them at some future date. Acts as a convenience wrapper around sqlite, currently, providing a key/value store accessed by 'domain'. """ # Copyright (C) 2007 Richard Purdie # Copyright (C) 2010 Chris Larson # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import collections import logging import os.path import sys import warnings from bb.compat import total_ordering from collections import Mapping try: import sqlite3 except ImportError: from pysqlite2 import dbapi2 as sqlite3 sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): raise Exception("sqlite3 version 3.3.0 or later is required.") logger = logging.getLogger("BitBake.PersistData") if hasattr(sqlite3, 'enable_shared_cache'): try: sqlite3.enable_shared_cache(True) except sqlite3.OperationalError: pass @total_ordering class SQLTable(collections.MutableMapping): """Object representing a table/domain in the database""" def __init__(self, cachefile, table): self.cachefile = cachefile self.table = table self.cursor = connect(self.cachefile) self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % table) def _execute(self, *query): """Execute a query, waiting to acquire a lock if necessary""" count = 0 while True: try: return self.cursor.execute(*query) except sqlite3.OperationalError as exc: if 'database is locked' in str(exc) and count < 500: count = count + 1 self.cursor.close() self.cursor = connect(self.cachefile) continue raise def __enter__(self): self.cursor.__enter__() return self def __exit__(self, *excinfo): self.cursor.__exit__(*excinfo) def __getitem__(self, key): data = self._execute("SELECT * from %s where key=?;" % self.table, [key]) for row in data: return row[1] raise KeyError(key) def __delitem__(self, key): if key not in self: raise KeyError(key) self._execute("DELETE from %s where key=?;" % self.table, [key]) def __setitem__(self, key, value): if not isinstance(key, basestring): raise TypeError('Only string keys are supported') elif not isinstance(value, basestring): raise TypeError('Only string values are supported') data = self._execute("SELECT * from %s where key=?;" % self.table, [key]) exists = len(list(data)) if exists: self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key]) else: self._execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value]) def __contains__(self, key): return key in set(self) def __len__(self): data = self._execute("SELECT COUNT(key) FROM %s;" % self.table) for row in data: return row[0] def __iter__(self): data = self._execute("SELECT key FROM %s;" % self.table) return (row[0] for row in data) def __lt__(self, other): if not isinstance(other, Mapping): raise NotImplemented return len(self) < len(other) def get_by_pattern(self, pattern): data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" % self.table, [pattern]) return [row[1] for row in data] def values(self): return list(self.itervalues()) def itervalues(self): data = self._execute("SELECT value FROM %s;" % self.table) return (row[0] for row in data) def items(self): return list(self.iteritems()) def iteritems(self): return self._execute("SELECT * FROM %s;" % self.table) def clear(self): self._execute("DELETE FROM %s;" % self.table) def has_key(self, key): return key in self class PersistData(object): """Deprecated representation of the bitbake persistent data store""" def __init__(self, d): warnings.warn("Use of PersistData is deprecated. Please use " "persist(domain, d) instead.", category=DeprecationWarning, stacklevel=2) self.data = persist(d) logger.debug(1, "Using '%s' as the persistent data cache", self.data.filename) def addDomain(self, domain): """ Add a domain (pending deprecation) """ return self.data[domain] def delDomain(self, domain): """ Removes a domain and all the data it contains """ del self.data[domain] def getKeyValues(self, domain): """ Return a list of key + value pairs for a domain """ return self.data[domain].items() def getValue(self, domain, key): """ Return the value of a key for a domain """ return self.data[domain][key] def setValue(self, domain, key, value): """ Sets the value of a key for a domain """ self.data[domain][key] = value def delValue(self, domain, key): """ Deletes a key/value pair """ del self.data[domain][key] def connect(database): connection = sqlite3.connect(database, timeout=5, isolation_level=None) connection.execute("pragma synchronous = off;") return connection def persist(domain, d): """Convenience factory for SQLTable objects based upon metadata""" import bb.utils cachedir = (d.getVar("PERSISTENT_DIR", True) or d.getVar("CACHE", True)) if not cachedir: logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable") sys.exit(1) bb.utils.mkdirhier(cachedir) cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3") return SQLTable(cachefile, domain)