diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-07-24 09:13:55 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-08-06 11:21:31 +0100 |
commit | ca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8 (patch) | |
tree | ee5fe5d5e780cd965bbe7f9a0d25d91d280b1834 /bitbake | |
parent | d9aafb85072bef3b5baa54408969d54a8425a111 (diff) | |
download | poky-ca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8.tar.gz |
bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE
Its useful, particularly in the local developer model of usage, for
bitbake to start and stop a hash equivalence server on local port,
rather than relying on one being started by the user before the build.
The new BB_HASHSERVE variable supports this.
The database handling is moved internally into the hashserv code so that
different threads/processes can be used for the server without errors.
(Bitbake rev: a4fa8f1bd88995ae60e10430316fbed63d478587)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rwxr-xr-x | bitbake/bin/bitbake-hashserv | 4 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-worker | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 15 | ||||
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 1 | ||||
-rw-r--r-- | bitbake/lib/hashserv/__init__.py | 19 | ||||
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 7 |
6 files changed, 37 insertions, 10 deletions
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv index 848a0ca39f..6c911c098a 100755 --- a/bitbake/bin/bitbake-hashserv +++ b/bitbake/bin/bitbake-hashserv | |||
@@ -41,9 +41,7 @@ def main(): | |||
41 | console.setLevel(level) | 41 | console.setLevel(level) |
42 | logger.addHandler(console) | 42 | logger.addHandler(console) |
43 | 43 | ||
44 | db = sqlite3.connect(args.database) | 44 | server = hashserv.create_server((args.address, args.port), args.database, args.prefix) |
45 | |||
46 | server = hashserv.create_server((args.address, args.port), db, args.prefix) | ||
47 | server.serve_forever() | 45 | server.serve_forever() |
48 | return 0 | 46 | return 0 |
49 | 47 | ||
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker index 3e502d5ca9..96369199f2 100755 --- a/bitbake/bin/bitbake-worker +++ b/bitbake/bin/bitbake-worker | |||
@@ -418,6 +418,7 @@ class BitbakeWorker(object): | |||
418 | bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] | 418 | bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] |
419 | for mc in self.databuilder.mcdata: | 419 | for mc in self.databuilder.mcdata: |
420 | self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"]) | 420 | self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"]) |
421 | self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservport"]) | ||
421 | 422 | ||
422 | def handle_newtaskhashes(self, data): | 423 | def handle_newtaskhashes(self, data): |
423 | self.workerdata["newhashes"] = pickle.loads(data) | 424 | self.workerdata["newhashes"] = pickle.loads(data) |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index b4851e13ed..84f25ec94e 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -31,6 +31,7 @@ import pyinotify | |||
31 | import json | 31 | import json |
32 | import pickle | 32 | import pickle |
33 | import codecs | 33 | import codecs |
34 | import hashserv | ||
34 | 35 | ||
35 | logger = logging.getLogger("BitBake") | 36 | logger = logging.getLogger("BitBake") |
36 | collectlog = logging.getLogger("BitBake.Collection") | 37 | collectlog = logging.getLogger("BitBake.Collection") |
@@ -230,6 +231,8 @@ class BBCooker: | |||
230 | self.state = state.initial | 231 | self.state = state.initial |
231 | 232 | ||
232 | self.parser = None | 233 | self.parser = None |
234 | self.hashserv = None | ||
235 | self.hashservport = None | ||
233 | 236 | ||
234 | signal.signal(signal.SIGTERM, self.sigterm_exception) | 237 | signal.signal(signal.SIGTERM, self.sigterm_exception) |
235 | # Let SIGHUP exit as SIGTERM | 238 | # Let SIGHUP exit as SIGTERM |
@@ -368,6 +371,15 @@ class BBCooker: | |||
368 | 371 | ||
369 | self.data.setVar('BB_CMDLINE', self.ui_cmdline) | 372 | self.data.setVar('BB_CMDLINE', self.ui_cmdline) |
370 | 373 | ||
374 | if self.data.getVar("BB_HASHSERVE") == "localhost:0": | ||
375 | dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" | ||
376 | self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '') | ||
377 | self.hashservport = "localhost:" + str(self.hashserv.server_port) | ||
378 | thread = threading.Thread(target=self.hashserv.serve_forever) | ||
379 | thread.daemon = True | ||
380 | thread.start() | ||
381 | self.data.setVar("BB_HASHSERVE", self.hashservport) | ||
382 | |||
371 | # | 383 | # |
372 | # Copy of the data store which has been expanded. | 384 | # Copy of the data store which has been expanded. |
373 | # Used for firing events and accessing variables where expansion needs to be accounted for | 385 | # Used for firing events and accessing variables where expansion needs to be accounted for |
@@ -1645,9 +1657,10 @@ class BBCooker: | |||
1645 | 1657 | ||
1646 | def post_serve(self): | 1658 | def post_serve(self): |
1647 | prserv.serv.auto_shutdown() | 1659 | prserv.serv.auto_shutdown() |
1660 | if self.hashserv: | ||
1661 | self.hashserv.shutdown() | ||
1648 | bb.event.fire(CookerExit(), self.data) | 1662 | bb.event.fire(CookerExit(), self.data) |
1649 | 1663 | ||
1650 | |||
1651 | def shutdown(self, force = False): | 1664 | def shutdown(self, force = False): |
1652 | if force: | 1665 | if force: |
1653 | self.state = state.forceshutdown | 1666 | self.state = state.forceshutdown |
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 05c1200da1..82cc9af81b 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1254,6 +1254,7 @@ class RunQueue: | |||
1254 | "buildname" : self.cfgData.getVar("BUILDNAME"), | 1254 | "buildname" : self.cfgData.getVar("BUILDNAME"), |
1255 | "date" : self.cfgData.getVar("DATE"), | 1255 | "date" : self.cfgData.getVar("DATE"), |
1256 | "time" : self.cfgData.getVar("TIME"), | 1256 | "time" : self.cfgData.getVar("TIME"), |
1257 | "hashservport" : self.cooker.hashservport, | ||
1257 | } | 1258 | } |
1258 | 1259 | ||
1259 | worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>") | 1260 | worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>") |
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py index 768c5504cf..7ec9b64419 100644 --- a/bitbake/lib/hashserv/__init__.py +++ b/bitbake/lib/hashserv/__init__.py | |||
@@ -18,8 +18,15 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): | |||
18 | def log_message(self, f, *args): | 18 | def log_message(self, f, *args): |
19 | logger.debug(f, *args) | 19 | logger.debug(f, *args) |
20 | 20 | ||
21 | def opendb(self): | ||
22 | self.db = sqlite3.connect(self.dbname) | ||
23 | self.db.row_factory = sqlite3.Row | ||
24 | |||
21 | def do_GET(self): | 25 | def do_GET(self): |
22 | try: | 26 | try: |
27 | if not self.db: | ||
28 | self.opendb() | ||
29 | |||
23 | p = urllib.parse.urlparse(self.path) | 30 | p = urllib.parse.urlparse(self.path) |
24 | 31 | ||
25 | if p.path != self.prefix + '/v1/equivalent': | 32 | if p.path != self.prefix + '/v1/equivalent': |
@@ -52,6 +59,9 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): | |||
52 | 59 | ||
53 | def do_POST(self): | 60 | def do_POST(self): |
54 | try: | 61 | try: |
62 | if not self.db: | ||
63 | self.opendb() | ||
64 | |||
55 | p = urllib.parse.urlparse(self.path) | 65 | p = urllib.parse.urlparse(self.path) |
56 | 66 | ||
57 | if p.path != self.prefix + '/v1/equivalent': | 67 | if p.path != self.prefix + '/v1/equivalent': |
@@ -123,14 +133,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): | |||
123 | self.send_error(400, explain=traceback.format_exc()) | 133 | self.send_error(400, explain=traceback.format_exc()) |
124 | return | 134 | return |
125 | 135 | ||
126 | def create_server(addr, db, prefix=''): | 136 | def create_server(addr, dbname, prefix=''): |
127 | class Handler(HashEquivalenceServer): | 137 | class Handler(HashEquivalenceServer): |
128 | pass | 138 | pass |
129 | 139 | ||
130 | Handler.prefix = prefix | 140 | db = sqlite3.connect(dbname) |
131 | Handler.db = db | ||
132 | db.row_factory = sqlite3.Row | 141 | db.row_factory = sqlite3.Row |
133 | 142 | ||
143 | Handler.prefix = prefix | ||
144 | Handler.db = None | ||
145 | Handler.dbname = dbname | ||
146 | |||
134 | with contextlib.closing(db.cursor()) as cursor: | 147 | with contextlib.closing(db.cursor()) as cursor: |
135 | cursor.execute(''' | 148 | cursor.execute(''' |
136 | CREATE TABLE IF NOT EXISTS tasks_v2 ( | 149 | CREATE TABLE IF NOT EXISTS tasks_v2 ( |
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index 8300a25599..15f4a52ee9 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -11,14 +11,15 @@ import sqlite3 | |||
11 | import hashlib | 11 | import hashlib |
12 | import urllib.request | 12 | import urllib.request |
13 | import json | 13 | import json |
14 | import tempfile | ||
14 | from . import create_server | 15 | from . import create_server |
15 | 16 | ||
16 | class TestHashEquivalenceServer(unittest.TestCase): | 17 | class TestHashEquivalenceServer(unittest.TestCase): |
17 | def setUp(self): | 18 | def setUp(self): |
18 | # Start an in memory hash equivalence server in the background bound to | 19 | # Start a hash equivalence server in the background bound to |
19 | # an ephemeral port | 20 | # an ephemeral port |
20 | db = sqlite3.connect(':memory:', check_same_thread=False) | 21 | self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-") |
21 | self.server = create_server(('localhost', 0), db) | 22 | self.server = create_server(('localhost', 0), self.dbfile.name) |
22 | self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1] | 23 | self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1] |
23 | self.server_thread = threading.Thread(target=self.server.serve_forever) | 24 | self.server_thread = threading.Thread(target=self.server.serve_forever) |
24 | self.server_thread.start() | 25 | self.server_thread.start() |