diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2020-11-10 08:59:56 -0600 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-11-24 15:26:12 +0000 |
commit | 96b548a79d87120655da3ac5501b8ad4726cf1a4 (patch) | |
tree | 06938cfe533173ad02664dc2f187867a165c5871 /bitbake/lib/hashserv/__init__.py | |
parent | 859f43e176dcaaa652e24a2289abd75e18c077cf (diff) | |
download | poky-96b548a79d87120655da3ac5501b8ad4726cf1a4.tar.gz |
bitbake: bitbake: hashserve: Add support for readonly upstream
Adds support for an upstream server to be specified. The upstream server
will be queried for equivalent hashes whenever a miss is found in the
local server. If the server returns a match, it is merged into the
local database. In order to keep the get stream queries as fast as
possible since they are the critical path when bitbake is preparing the
run queue, missing tasks provided by the server are not immediately
pulled from the upstream server, but instead are put into a queue to be
backfilled by a worker task later.
(Bitbake rev: e6d6c0b39393e9bdf378c1eba141f815e26b724b)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/hashserv/__init__.py')
-rw-r--r-- | bitbake/lib/hashserv/__init__.py | 39 |
1 files changed, 22 insertions, 17 deletions
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py index 622ca17a91..55f48410d3 100644 --- a/bitbake/lib/hashserv/__init__.py +++ b/bitbake/lib/hashserv/__init__.py | |||
@@ -22,6 +22,24 @@ ADDR_TYPE_TCP = 1 | |||
22 | # is necessary | 22 | # is necessary |
23 | DEFAULT_MAX_CHUNK = 32 * 1024 | 23 | DEFAULT_MAX_CHUNK = 32 * 1024 |
24 | 24 | ||
25 | TABLE_DEFINITION = ( | ||
26 | ("method", "TEXT NOT NULL"), | ||
27 | ("outhash", "TEXT NOT NULL"), | ||
28 | ("taskhash", "TEXT NOT NULL"), | ||
29 | ("unihash", "TEXT NOT NULL"), | ||
30 | ("created", "DATETIME"), | ||
31 | |||
32 | # Optional fields | ||
33 | ("owner", "TEXT"), | ||
34 | ("PN", "TEXT"), | ||
35 | ("PV", "TEXT"), | ||
36 | ("PR", "TEXT"), | ||
37 | ("task", "TEXT"), | ||
38 | ("outhash_siginfo", "TEXT"), | ||
39 | ) | ||
40 | |||
41 | TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION) | ||
42 | |||
25 | def setup_database(database, sync=True): | 43 | def setup_database(database, sync=True): |
26 | db = sqlite3.connect(database) | 44 | db = sqlite3.connect(database) |
27 | db.row_factory = sqlite3.Row | 45 | db.row_factory = sqlite3.Row |
@@ -30,23 +48,10 @@ def setup_database(database, sync=True): | |||
30 | cursor.execute(''' | 48 | cursor.execute(''' |
31 | CREATE TABLE IF NOT EXISTS tasks_v2 ( | 49 | CREATE TABLE IF NOT EXISTS tasks_v2 ( |
32 | id INTEGER PRIMARY KEY AUTOINCREMENT, | 50 | id INTEGER PRIMARY KEY AUTOINCREMENT, |
33 | method TEXT NOT NULL, | 51 | %s |
34 | outhash TEXT NOT NULL, | ||
35 | taskhash TEXT NOT NULL, | ||
36 | unihash TEXT NOT NULL, | ||
37 | created DATETIME, | ||
38 | |||
39 | -- Optional fields | ||
40 | owner TEXT, | ||
41 | PN TEXT, | ||
42 | PV TEXT, | ||
43 | PR TEXT, | ||
44 | task TEXT, | ||
45 | outhash_siginfo TEXT, | ||
46 | |||
47 | UNIQUE(method, outhash, taskhash) | 52 | UNIQUE(method, outhash, taskhash) |
48 | ) | 53 | ) |
49 | ''') | 54 | ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION)) |
50 | cursor.execute('PRAGMA journal_mode = WAL') | 55 | cursor.execute('PRAGMA journal_mode = WAL') |
51 | cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF')) | 56 | cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF')) |
52 | 57 | ||
@@ -89,10 +94,10 @@ def chunkify(msg, max_chunk): | |||
89 | yield "\n" | 94 | yield "\n" |
90 | 95 | ||
91 | 96 | ||
92 | def create_server(addr, dbname, *, sync=True): | 97 | def create_server(addr, dbname, *, sync=True, upstream=None): |
93 | from . import server | 98 | from . import server |
94 | db = setup_database(dbname, sync=sync) | 99 | db = setup_database(dbname, sync=sync) |
95 | s = server.Server(db) | 100 | s = server.Server(db, upstream=upstream) |
96 | 101 | ||
97 | (typ, a) = parse_address(addr) | 102 | (typ, a) = parse_address(addr) |
98 | if typ == ADDR_TYPE_UNIX: | 103 | if typ == ADDR_TYPE_UNIX: |