summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xbitbake/bin/bitbake-hashserv67
-rwxr-xr-xbitbake/bin/bitbake-selftest2
-rw-r--r--bitbake/lib/hashserv/__init__.py152
-rw-r--r--bitbake/lib/hashserv/tests.py141
4 files changed, 362 insertions, 0 deletions
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv
new file mode 100755
index 0000000000..c49397b73a
--- /dev/null
+++ b/bitbake/bin/bitbake-hashserv
@@ -0,0 +1,67 @@
1#! /usr/bin/env python3
2#
3# Copyright (C) 2018 Garmin Ltd.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17import os
18import sys
19import logging
20import argparse
21import sqlite3
22
23sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
24
25import hashserv
26
27VERSION = "1.0.0"
28
29DEFAULT_HOST = ''
30DEFAULT_PORT = 8686
31
32def main():
33 parser = argparse.ArgumentParser(description='HTTP Equivalence Reference Server. Version=%s' % VERSION)
34 parser.add_argument('--address', default=DEFAULT_HOST, help='Bind address (default "%(default)s")')
35 parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Bind port (default %(default)d)')
36 parser.add_argument('--prefix', default='', help='HTTP path prefix (default "%(default)s")')
37 parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")')
38 parser.add_argument('--log', default='WARNING', help='Set logging level')
39
40 args = parser.parse_args()
41
42 logger = logging.getLogger('hashserv')
43
44 level = getattr(logging, args.log.upper(), None)
45 if not isinstance(level, int):
46 raise ValueError('Invalid log level: %s' % args.log)
47
48 logger.setLevel(level)
49 console = logging.StreamHandler()
50 console.setLevel(level)
51 logger.addHandler(console)
52
53 db = sqlite3.connect(args.database)
54
55 server = hashserv.create_server((args.address, args.port), db, args.prefix)
56 server.serve_forever()
57 return 0
58
59if __name__ == '__main__':
60 try:
61 ret = main()
62 except Exception:
63 ret = 1
64 import traceback
65 traceback.print_exc()
66 sys.exit(ret)
67
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index c970dcae90..99f1af910f 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
22import unittest 22import unittest
23try: 23try:
24 import bb 24 import bb
25 import hashserv
25 import layerindexlib 26 import layerindexlib
26except RuntimeError as exc: 27except RuntimeError as exc:
27 sys.exit(str(exc)) 28 sys.exit(str(exc))
@@ -35,6 +36,7 @@ tests = ["bb.tests.codeparser",
35 "bb.tests.parse", 36 "bb.tests.parse",
36 "bb.tests.persist_data", 37 "bb.tests.persist_data",
37 "bb.tests.utils", 38 "bb.tests.utils",
39 "hashserv.tests",
38 "layerindexlib.tests.layerindexobj", 40 "layerindexlib.tests.layerindexobj",
39 "layerindexlib.tests.restapi", 41 "layerindexlib.tests.restapi",
40 "layerindexlib.tests.cooker"] 42 "layerindexlib.tests.cooker"]
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py
new file mode 100644
index 0000000000..46bca7cab3
--- /dev/null
+++ b/bitbake/lib/hashserv/__init__.py
@@ -0,0 +1,152 @@
1# Copyright (C) 2018 Garmin Ltd.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License along
13# with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
15
16from http.server import BaseHTTPRequestHandler, HTTPServer
17import contextlib
18import urllib.parse
19import sqlite3
20import json
21import traceback
22import logging
23from datetime import datetime
24
25logger = logging.getLogger('hashserv')
26
27class HashEquivalenceServer(BaseHTTPRequestHandler):
28 def log_message(self, f, *args):
29 logger.debug(f, *args)
30
31 def do_GET(self):
32 try:
33 p = urllib.parse.urlparse(self.path)
34
35 if p.path != self.prefix + '/v1/equivalent':
36 self.send_error(404)
37 return
38
39 query = urllib.parse.parse_qs(p.query, strict_parsing=True)
40 method = query['method'][0]
41 taskhash = query['taskhash'][0]
42
43 d = None
44 with contextlib.closing(self.db.cursor()) as cursor:
45 cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1',
46 {'method': method, 'taskhash': taskhash})
47
48 row = cursor.fetchone()
49
50 if row is not None:
51 logger.debug('Found equivalent task %s', row['taskhash'])
52 d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
53
54 self.send_response(200)
55 self.send_header('Content-Type', 'application/json; charset=utf-8')
56 self.end_headers()
57 self.wfile.write(json.dumps(d).encode('utf-8'))
58 except:
59 logger.exception('Error in GET')
60 self.send_error(400, explain=traceback.format_exc())
61 return
62
63 def do_POST(self):
64 try:
65 p = urllib.parse.urlparse(self.path)
66
67 if p.path != self.prefix + '/v1/equivalent':
68 self.send_error(404)
69 return
70
71 length = int(self.headers['content-length'])
72 data = json.loads(self.rfile.read(length).decode('utf-8'))
73
74 with contextlib.closing(self.db.cursor()) as cursor:
75 cursor.execute('''
76 SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND outhash=:outhash
77 ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
78 created ASC
79 LIMIT 1
80 ''', {k: data[k] for k in ('method', 'outhash', 'taskhash')})
81
82 row = cursor.fetchone()
83
84 if row is None or row['taskhash'] != data['taskhash']:
85 unihash = data['unihash']
86 if row is not None:
87 unihash = row['unihash']
88
89 insert_data = {
90 'method': data['method'],
91 'outhash': data['outhash'],
92 'taskhash': data['taskhash'],
93 'unihash': unihash,
94 'created': datetime.now()
95 }
96
97 for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
98 if k in data:
99 insert_data[k] = data[k]
100
101 cursor.execute('''INSERT INTO tasks_v1 (%s) VALUES (%s)''' % (
102 ', '.join(sorted(insert_data.keys())),
103 ', '.join(':' + k for k in sorted(insert_data.keys()))),
104 insert_data)
105
106 logger.info('Adding taskhash %s with unihash %s', data['taskhash'], unihash)
107 cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE id=:id', {'id': cursor.lastrowid})
108 row = cursor.fetchone()
109
110 self.db.commit()
111
112 d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
113
114 self.send_response(200)
115 self.send_header('Content-Type', 'application/json; charset=utf-8')
116 self.end_headers()
117 self.wfile.write(json.dumps(d).encode('utf-8'))
118 except:
119 logger.exception('Error in POST')
120 self.send_error(400, explain=traceback.format_exc())
121 return
122
123def create_server(addr, db, prefix=''):
124 class Handler(HashEquivalenceServer):
125 pass
126
127 Handler.prefix = prefix
128 Handler.db = db
129 db.row_factory = sqlite3.Row
130
131 with contextlib.closing(db.cursor()) as cursor:
132 cursor.execute('''
133 CREATE TABLE IF NOT EXISTS tasks_v1 (
134 id INTEGER PRIMARY KEY AUTOINCREMENT,
135 method TEXT NOT NULL,
136 outhash TEXT NOT NULL,
137 taskhash TEXT NOT NULL,
138 unihash TEXT NOT NULL,
139 created DATETIME,
140
141 -- Optional fields
142 owner TEXT,
143 PN TEXT,
144 PV TEXT,
145 PR TEXT,
146 task TEXT,
147 outhash_siginfo TEXT
148 )
149 ''')
150
151 logger.info('Starting server on %s', addr)
152 return HTTPServer(addr, Handler)
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py
new file mode 100644
index 0000000000..806b54c5eb
--- /dev/null
+++ b/bitbake/lib/hashserv/tests.py
@@ -0,0 +1,141 @@
1#! /usr/bin/env python3
2#
3# Copyright (C) 2018 Garmin Ltd.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import unittest
19import threading
20import sqlite3
21import hashlib
22import urllib.request
23import json
24from . import create_server
25
26class TestHashEquivalenceServer(unittest.TestCase):
27 def setUp(self):
28 # Start an in memory hash equivalence server in the background bound to
29 # an ephemeral port
30 db = sqlite3.connect(':memory:', check_same_thread=False)
31 self.server = create_server(('localhost', 0), db)
32 self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
33 self.server_thread = threading.Thread(target=self.server.serve_forever)
34 self.server_thread.start()
35
36 def tearDown(self):
37 # Shutdown server
38 s = getattr(self, 'server', None)
39 if s is not None:
40 self.server.shutdown()
41 self.server_thread.join()
42 self.server.server_close()
43
44 def send_get(self, path):
45 url = '%s/%s' % (self.server_addr, path)
46 request = urllib.request.Request(url)
47 response = urllib.request.urlopen(request)
48 return json.loads(response.read().decode('utf-8'))
49
50 def send_post(self, path, data):
51 headers = {'content-type': 'application/json'}
52 url = '%s/%s' % (self.server_addr, path)
53 request = urllib.request.Request(url, json.dumps(data).encode('utf-8'), headers)
54 response = urllib.request.urlopen(request)
55 return json.loads(response.read().decode('utf-8'))
56
57 def test_create_hash(self):
58 # Simple test that hashes can be created
59 taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
60 outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
61 unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
62
63 d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
64 self.assertIsNone(d, msg='Found unexpected task, %r' % d)
65
66 d = self.send_post('v1/equivalent', {
67 'taskhash': taskhash,
68 'method': 'TestMethod',
69 'outhash': outhash,
70 'unihash': unihash,
71 })
72 self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
73
74 def test_create_equivalent(self):
75 # Tests that a second reported task with the same outhash will be
76 # assigned the same unihash
77 taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
78 outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
79 unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
80 d = self.send_post('v1/equivalent', {
81 'taskhash': taskhash,
82 'method': 'TestMethod',
83 'outhash': outhash,
84 'unihash': unihash,
85 })
86 self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
87
88 # Report a different task with the same outhash. The returned unihash
89 # should match the first task
90 taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
91 unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
92 d = self.send_post('v1/equivalent', {
93 'taskhash': taskhash2,
94 'method': 'TestMethod',
95 'outhash': outhash,
96 'unihash': unihash2,
97 })
98 self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash')
99
100 def test_duplicate_taskhash(self):
101 # Tests that duplicate reports of the same taskhash with different
102 # outhash & unihash always return the unihash from the first reported
103 # taskhash
104 taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
105 outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
106 unihash = '218e57509998197d570e2c98512d0105985dffc9'
107 d = self.send_post('v1/equivalent', {
108 'taskhash': taskhash,
109 'method': 'TestMethod',
110 'outhash': outhash,
111 'unihash': unihash,
112 })
113
114 d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
115 self.assertEqual(d['unihash'], unihash)
116
117 outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
118 unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
119 d = self.send_post('v1/equivalent', {
120 'taskhash': taskhash,
121 'method': 'TestMethod',
122 'outhash': outhash2,
123 'unihash': unihash2
124 })
125
126 d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
127 self.assertEqual(d['unihash'], unihash)
128
129 outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
130 unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
131 d = self.send_post('v1/equivalent', {
132 'taskhash': taskhash,
133 'method': 'TestMethod',
134 'outhash': outhash3,
135 'unihash': unihash3
136 })
137
138 d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash)
139 self.assertEqual(d['unihash'], unihash)
140
141