diff options
Diffstat (limited to 'bitbake/lib/hashserv/tests.py')
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 147 |
1 files changed, 115 insertions, 32 deletions
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index 4566f24738..3dd9a31bee 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -16,35 +16,54 @@ import threading | |||
16 | import unittest | 16 | import unittest |
17 | import socket | 17 | import socket |
18 | 18 | ||
19 | def _run_server(server, idx): | ||
20 | # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', | ||
21 | # format='%(levelname)s %(filename)s:%(lineno)d %(message)s') | ||
22 | sys.stdout = open('bbhashserv-%d.log' % idx, 'w') | ||
23 | sys.stderr = sys.stdout | ||
24 | server.serve_forever() | ||
19 | 25 | ||
20 | class TestHashEquivalenceServer(object): | 26 | class TestHashEquivalenceServer(object): |
21 | METHOD = 'TestMethod' | 27 | METHOD = 'TestMethod' |
22 | 28 | ||
23 | def _run_server(self): | 29 | server_index = 0 |
24 | # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', | 30 | |
25 | # format='%(levelname)s %(filename)s:%(lineno)d %(message)s') | 31 | def start_server(self, dbpath=None, upstream=None): |
26 | self.server.serve_forever() | 32 | self.server_index += 1 |
33 | if dbpath is None: | ||
34 | dbpath = os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index) | ||
35 | |||
36 | def cleanup_thread(thread): | ||
37 | thread.terminate() | ||
38 | thread.join() | ||
39 | |||
40 | server = create_server(self.get_server_addr(self.server_index), dbpath, upstream=upstream) | ||
41 | server.dbpath = dbpath | ||
42 | |||
43 | server.thread = multiprocessing.Process(target=_run_server, args=(server, self.server_index)) | ||
44 | server.thread.start() | ||
45 | self.addCleanup(cleanup_thread, server.thread) | ||
46 | |||
47 | def cleanup_client(client): | ||
48 | client.close() | ||
49 | |||
50 | client = create_client(server.address) | ||
51 | self.addCleanup(cleanup_client, client) | ||
52 | |||
53 | return (client, server) | ||
27 | 54 | ||
28 | def setUp(self): | 55 | def setUp(self): |
29 | if sys.version_info < (3, 5, 0): | 56 | if sys.version_info < (3, 5, 0): |
30 | self.skipTest('Python 3.5 or later required') | 57 | self.skipTest('Python 3.5 or later required') |
31 | 58 | ||
32 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') | 59 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') |
33 | self.dbfile = os.path.join(self.temp_dir.name, 'db.sqlite') | 60 | self.addCleanup(self.temp_dir.cleanup) |
34 | 61 | ||
35 | self.server = create_server(self.get_server_addr(), self.dbfile) | 62 | (self.client, self.server) = self.start_server() |
36 | self.server_thread = multiprocessing.Process(target=self._run_server) | 63 | |
37 | self.server_thread.start() | 64 | def assertClientGetHash(self, client, taskhash, unihash): |
38 | self.client = create_client(self.server.address) | 65 | result = client.get_unihash(self.METHOD, taskhash) |
39 | 66 | self.assertEqual(result, unihash) | |
40 | def tearDown(self): | ||
41 | # Shutdown server | ||
42 | s = getattr(self, 'server', None) | ||
43 | if s is not None: | ||
44 | self.server_thread.terminate() | ||
45 | self.server_thread.join() | ||
46 | self.client.close() | ||
47 | self.temp_dir.cleanup() | ||
48 | 67 | ||
49 | def test_create_hash(self): | 68 | def test_create_hash(self): |
50 | # Simple test that hashes can be created | 69 | # Simple test that hashes can be created |
@@ -52,8 +71,7 @@ class TestHashEquivalenceServer(object): | |||
52 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' | 71 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' |
53 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' | 72 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' |
54 | 73 | ||
55 | result = self.client.get_unihash(self.METHOD, taskhash) | 74 | self.assertClientGetHash(self.client, taskhash, None) |
56 | self.assertIsNone(result, msg='Found unexpected task, %r' % result) | ||
57 | 75 | ||
58 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | 76 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
59 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') | 77 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') |
@@ -84,22 +102,19 @@ class TestHashEquivalenceServer(object): | |||
84 | unihash = '218e57509998197d570e2c98512d0105985dffc9' | 102 | unihash = '218e57509998197d570e2c98512d0105985dffc9' |
85 | self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | 103 | self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
86 | 104 | ||
87 | result = self.client.get_unihash(self.METHOD, taskhash) | 105 | self.assertClientGetHash(self.client, taskhash, unihash) |
88 | self.assertEqual(result, unihash) | ||
89 | 106 | ||
90 | outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' | 107 | outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' |
91 | unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' | 108 | unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' |
92 | self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2) | 109 | self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2) |
93 | 110 | ||
94 | result = self.client.get_unihash(self.METHOD, taskhash) | 111 | self.assertClientGetHash(self.client, taskhash, unihash) |
95 | self.assertEqual(result, unihash) | ||
96 | 112 | ||
97 | outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' | 113 | outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' |
98 | unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' | 114 | unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' |
99 | self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3) | 115 | self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3) |
100 | 116 | ||
101 | result = self.client.get_unihash(self.METHOD, taskhash) | 117 | self.assertClientGetHash(self.client, taskhash, unihash) |
102 | self.assertEqual(result, unihash) | ||
103 | 118 | ||
104 | def test_huge_message(self): | 119 | def test_huge_message(self): |
105 | # Simple test that hashes can be created | 120 | # Simple test that hashes can be created |
@@ -107,8 +122,7 @@ class TestHashEquivalenceServer(object): | |||
107 | outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44' | 122 | outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44' |
108 | unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824' | 123 | unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824' |
109 | 124 | ||
110 | result = self.client.get_unihash(self.METHOD, taskhash) | 125 | self.assertClientGetHash(self.client, taskhash, None) |
111 | self.assertIsNone(result, msg='Found unexpected task, %r' % result) | ||
112 | 126 | ||
113 | siginfo = "0" * (self.client.max_chunk * 4) | 127 | siginfo = "0" * (self.client.max_chunk * 4) |
114 | 128 | ||
@@ -156,14 +170,83 @@ class TestHashEquivalenceServer(object): | |||
156 | 170 | ||
157 | self.assertFalse(failures) | 171 | self.assertFalse(failures) |
158 | 172 | ||
173 | def test_upstream_server(self): | ||
174 | # Tests upstream server support. This is done by creating two servers | ||
175 | # that share a database file. The downstream server has it upstream | ||
176 | # set to the test server, whereas the side server doesn't. This allows | ||
177 | # verification that the hash requests are being proxied to the upstream | ||
178 | # server by verifying that they appear on the downstream client, but not | ||
179 | # the side client. It also verifies that the results are pulled into | ||
180 | # the downstream database by checking that the downstream and side servers | ||
181 | # match after the downstream is done waiting for all backfill tasks | ||
182 | (down_client, down_server) = self.start_server(upstream=self.server.address) | ||
183 | (side_client, side_server) = self.start_server(dbpath=down_server.dbpath) | ||
184 | |||
185 | def check_hash(taskhash, unihash, old_sidehash): | ||
186 | nonlocal down_client | ||
187 | nonlocal side_client | ||
188 | |||
189 | # check upstream server | ||
190 | self.assertClientGetHash(self.client, taskhash, unihash) | ||
191 | |||
192 | # Hash should *not* be present on the side server | ||
193 | self.assertClientGetHash(side_client, taskhash, old_sidehash) | ||
194 | |||
195 | # Hash should be present on the downstream server, since it | ||
196 | # will defer to the upstream server. This will trigger | ||
197 | # the backfill in the downstream server | ||
198 | self.assertClientGetHash(down_client, taskhash, unihash) | ||
199 | |||
200 | # After waiting for the downstream client to finish backfilling the | ||
201 | # task from the upstream server, it should appear in the side server | ||
202 | # since the database is populated | ||
203 | down_client.backfill_wait() | ||
204 | self.assertClientGetHash(side_client, taskhash, unihash) | ||
205 | |||
206 | # Basic report | ||
207 | taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' | ||
208 | outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' | ||
209 | unihash = '218e57509998197d570e2c98512d0105985dffc9' | ||
210 | self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | ||
211 | |||
212 | check_hash(taskhash, unihash, None) | ||
213 | |||
214 | # Duplicated taskhash with multiple output hashes and unihashes. | ||
215 | # All servers should agree with the originally reported hash | ||
216 | outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' | ||
217 | unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' | ||
218 | self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2) | ||
219 | |||
220 | check_hash(taskhash, unihash, unihash) | ||
221 | |||
222 | # Report an equivalent task. The sideload will originally report | ||
223 | # no unihash until backfilled | ||
224 | taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1" | ||
225 | unihash3 = "def64766090d28f627e816454ed46894bb3aab36" | ||
226 | self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3) | ||
227 | |||
228 | check_hash(taskhash3, unihash, None) | ||
229 | |||
230 | # Test that reporting a unihash in the downstream client isn't | ||
231 | # propagating to the upstream server | ||
232 | taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" | ||
233 | outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" | ||
234 | unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d" | ||
235 | down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4) | ||
236 | down_client.backfill_wait() | ||
237 | |||
238 | self.assertClientGetHash(down_client, taskhash4, unihash4) | ||
239 | self.assertClientGetHash(side_client, taskhash4, unihash4) | ||
240 | self.assertClientGetHash(self.client, taskhash4, None) | ||
241 | |||
159 | 242 | ||
160 | class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase): | 243 | class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase): |
161 | def get_server_addr(self): | 244 | def get_server_addr(self, server_idx): |
162 | return "unix://" + os.path.join(self.temp_dir.name, 'sock') | 245 | return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx) |
163 | 246 | ||
164 | 247 | ||
165 | class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase): | 248 | class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase): |
166 | def get_server_addr(self): | 249 | def get_server_addr(self, server_idx): |
167 | # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled. | 250 | # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled. |
168 | # If IPv6 is enabled, it should be safe to use localhost directly, in general | 251 | # If IPv6 is enabled, it should be safe to use localhost directly, in general |
169 | # case it is more reliable to resolve the IP address explicitly. | 252 | # case it is more reliable to resolve the IP address explicitly. |