diff options
Diffstat (limited to 'bitbake/lib/hashserv/tests.py')
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 159 |
1 files changed, 85 insertions, 74 deletions
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index 6845b53884..6584ff57b4 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -1,29 +1,40 @@ | |||
1 | #! /usr/bin/env python3 | 1 | #! /usr/bin/env python3 |
2 | # | 2 | # |
3 | # Copyright (C) 2018 Garmin Ltd. | 3 | # Copyright (C) 2018-2019 Garmin Ltd. |
4 | # | 4 | # |
5 | # SPDX-License-Identifier: GPL-2.0-only | 5 | # SPDX-License-Identifier: GPL-2.0-only |
6 | # | 6 | # |
7 | 7 | ||
8 | import unittest | 8 | from . import create_server, create_client |
9 | import multiprocessing | ||
10 | import sqlite3 | ||
11 | import hashlib | 9 | import hashlib |
12 | import urllib.request | 10 | import logging |
13 | import json | 11 | import multiprocessing |
12 | import sys | ||
14 | import tempfile | 13 | import tempfile |
15 | from . import create_server | 14 | import threading |
15 | import unittest | ||
16 | |||
17 | |||
18 | class TestHashEquivalenceServer(object): | ||
19 | METHOD = 'TestMethod' | ||
20 | |||
21 | def _run_server(self): | ||
22 | # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', | ||
23 | # format='%(levelname)s %(filename)s:%(lineno)d %(message)s') | ||
24 | self.server.serve_forever() | ||
16 | 25 | ||
17 | class TestHashEquivalenceServer(unittest.TestCase): | ||
18 | def setUp(self): | 26 | def setUp(self): |
19 | # Start a hash equivalence server in the background bound to | 27 | if sys.version_info < (3, 5, 0): |
20 | # an ephemeral port | 28 | self.skipTest('Python 3.5 or later required') |
21 | self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-") | 29 | |
22 | self.server = create_server(('localhost', 0), self.dbfile.name) | 30 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') |
23 | self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1] | 31 | self.dbfile = os.path.join(self.temp_dir.name, 'db.sqlite') |
24 | self.server_thread = multiprocessing.Process(target=self.server.serve_forever) | 32 | |
33 | self.server = create_server(self.get_server_addr(), self.dbfile) | ||
34 | self.server_thread = multiprocessing.Process(target=self._run_server) | ||
25 | self.server_thread.daemon = True | 35 | self.server_thread.daemon = True |
26 | self.server_thread.start() | 36 | self.server_thread.start() |
37 | self.client = create_client(self.server.address) | ||
27 | 38 | ||
28 | def tearDown(self): | 39 | def tearDown(self): |
29 | # Shutdown server | 40 | # Shutdown server |
@@ -31,19 +42,8 @@ class TestHashEquivalenceServer(unittest.TestCase): | |||
31 | if s is not None: | 42 | if s is not None: |
32 | self.server_thread.terminate() | 43 | self.server_thread.terminate() |
33 | self.server_thread.join() | 44 | self.server_thread.join() |
34 | 45 | self.client.close() | |
35 | def send_get(self, path): | 46 | self.temp_dir.cleanup() |
36 | url = '%s/%s' % (self.server_addr, path) | ||
37 | request = urllib.request.Request(url) | ||
38 | response = urllib.request.urlopen(request) | ||
39 | return json.loads(response.read().decode('utf-8')) | ||
40 | |||
41 | def send_post(self, path, data): | ||
42 | headers = {'content-type': 'application/json'} | ||
43 | url = '%s/%s' % (self.server_addr, path) | ||
44 | request = urllib.request.Request(url, json.dumps(data).encode('utf-8'), headers) | ||
45 | response = urllib.request.urlopen(request) | ||
46 | return json.loads(response.read().decode('utf-8')) | ||
47 | 47 | ||
48 | def test_create_hash(self): | 48 | def test_create_hash(self): |
49 | # Simple test that hashes can be created | 49 | # Simple test that hashes can be created |
@@ -51,16 +51,11 @@ class TestHashEquivalenceServer(unittest.TestCase): | |||
51 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' | 51 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' |
52 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' | 52 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' |
53 | 53 | ||
54 | d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) | 54 | result = self.client.get_unihash(self.METHOD, taskhash) |
55 | self.assertIsNone(d, msg='Found unexpected task, %r' % d) | 55 | self.assertIsNone(result, msg='Found unexpected task, %r' % result) |
56 | 56 | ||
57 | d = self.send_post('v1/equivalent', { | 57 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
58 | 'taskhash': taskhash, | 58 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') |
59 | 'method': 'TestMethod', | ||
60 | 'outhash': outhash, | ||
61 | 'unihash': unihash, | ||
62 | }) | ||
63 | self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') | ||
64 | 59 | ||
65 | def test_create_equivalent(self): | 60 | def test_create_equivalent(self): |
66 | # Tests that a second reported task with the same outhash will be | 61 | # Tests that a second reported task with the same outhash will be |
@@ -68,25 +63,16 @@ class TestHashEquivalenceServer(unittest.TestCase): | |||
68 | taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' | 63 | taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' |
69 | outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' | 64 | outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' |
70 | unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646' | 65 | unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646' |
71 | d = self.send_post('v1/equivalent', { | 66 | |
72 | 'taskhash': taskhash, | 67 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
73 | 'method': 'TestMethod', | 68 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') |
74 | 'outhash': outhash, | ||
75 | 'unihash': unihash, | ||
76 | }) | ||
77 | self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') | ||
78 | 69 | ||
79 | # Report a different task with the same outhash. The returned unihash | 70 | # Report a different task with the same outhash. The returned unihash |
80 | # should match the first task | 71 | # should match the first task |
81 | taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4' | 72 | taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4' |
82 | unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b' | 73 | unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b' |
83 | d = self.send_post('v1/equivalent', { | 74 | result = self.client.report_unihash(taskhash2, self.METHOD, outhash, unihash2) |
84 | 'taskhash': taskhash2, | 75 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') |
85 | 'method': 'TestMethod', | ||
86 | 'outhash': outhash, | ||
87 | 'unihash': unihash2, | ||
88 | }) | ||
89 | self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') | ||
90 | 76 | ||
91 | def test_duplicate_taskhash(self): | 77 | def test_duplicate_taskhash(self): |
92 | # Tests that duplicate reports of the same taskhash with different | 78 | # Tests that duplicate reports of the same taskhash with different |
@@ -95,38 +81,63 @@ class TestHashEquivalenceServer(unittest.TestCase): | |||
95 | taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' | 81 | taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' |
96 | outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' | 82 | outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' |
97 | unihash = '218e57509998197d570e2c98512d0105985dffc9' | 83 | unihash = '218e57509998197d570e2c98512d0105985dffc9' |
98 | d = self.send_post('v1/equivalent', { | 84 | self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
99 | 'taskhash': taskhash, | ||
100 | 'method': 'TestMethod', | ||
101 | 'outhash': outhash, | ||
102 | 'unihash': unihash, | ||
103 | }) | ||
104 | 85 | ||
105 | d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) | 86 | result = self.client.get_unihash(self.METHOD, taskhash) |
106 | self.assertEqual(d['unihash'], unihash) | 87 | self.assertEqual(result, unihash) |
107 | 88 | ||
108 | outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' | 89 | outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' |
109 | unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' | 90 | unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' |
110 | d = self.send_post('v1/equivalent', { | 91 | self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2) |
111 | 'taskhash': taskhash, | ||
112 | 'method': 'TestMethod', | ||
113 | 'outhash': outhash2, | ||
114 | 'unihash': unihash2 | ||
115 | }) | ||
116 | 92 | ||
117 | d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) | 93 | result = self.client.get_unihash(self.METHOD, taskhash) |
118 | self.assertEqual(d['unihash'], unihash) | 94 | self.assertEqual(result, unihash) |
119 | 95 | ||
120 | outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' | 96 | outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' |
121 | unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' | 97 | unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' |
122 | d = self.send_post('v1/equivalent', { | 98 | self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3) |
123 | 'taskhash': taskhash, | 99 | |
124 | 'method': 'TestMethod', | 100 | result = self.client.get_unihash(self.METHOD, taskhash) |
125 | 'outhash': outhash3, | 101 | self.assertEqual(result, unihash) |
126 | 'unihash': unihash3 | 102 | |
127 | }) | 103 | def test_stress(self): |
104 | def query_server(failures): | ||
105 | client = Client(self.server.address) | ||
106 | try: | ||
107 | for i in range(1000): | ||
108 | taskhash = hashlib.sha256() | ||
109 | taskhash.update(str(i).encode('utf-8')) | ||
110 | taskhash = taskhash.hexdigest() | ||
111 | result = client.get_unihash(self.METHOD, taskhash) | ||
112 | if result != taskhash: | ||
113 | failures.append("taskhash mismatch: %s != %s" % (result, taskhash)) | ||
114 | finally: | ||
115 | client.close() | ||
116 | |||
117 | # Report hashes | ||
118 | for i in range(1000): | ||
119 | taskhash = hashlib.sha256() | ||
120 | taskhash.update(str(i).encode('utf-8')) | ||
121 | taskhash = taskhash.hexdigest() | ||
122 | self.client.report_unihash(taskhash, self.METHOD, taskhash, taskhash) | ||
123 | |||
124 | failures = [] | ||
125 | threads = [threading.Thread(target=query_server, args=(failures,)) for t in range(100)] | ||
126 | |||
127 | for t in threads: | ||
128 | t.start() | ||
129 | |||
130 | for t in threads: | ||
131 | t.join() | ||
132 | |||
133 | self.assertFalse(failures) | ||
134 | |||
128 | 135 | ||
129 | d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) | 136 | class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase): |
130 | self.assertEqual(d['unihash'], unihash) | 137 | def get_server_addr(self): |
138 | return "unix://" + os.path.join(self.temp_dir.name, 'sock') | ||
131 | 139 | ||
132 | 140 | ||
141 | class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase): | ||
142 | def get_server_addr(self): | ||
143 | return "localhost:0" | ||