diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2024-02-18 15:59:50 -0700 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2024-02-19 11:58:12 +0000 |
commit | 37b4d7e4931cb032659273c19520d74083ffb0e9 (patch) | |
tree | e68b6d0905db327ab0da110670a5f57952774288 /bitbake/lib/hashserv | |
parent | 2406bd10550997ecd033baad93fcb59b223c5aa8 (diff) | |
download | poky-37b4d7e4931cb032659273c19520d74083ffb0e9.tar.gz |
bitbake: hashserv: Add Client Pool
Implements a Client Pool derived from the AsyncRPC client pool that
allows querying for multiple equivalent hashes in parallel
(Bitbake rev: ba4c764d8061c7b88cd4985ca493d6ea6e317106)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/hashserv')
-rw-r--r-- | bitbake/lib/hashserv/client.py | 80 | ||||
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 83 |
2 files changed, 163 insertions, 0 deletions
diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index daf1e12842..b269879ecf 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py | |||
@@ -283,3 +283,83 @@ class Client(bb.asyncrpc.Client): | |||
283 | 283 | ||
284 | def _get_async_client(self): | 284 | def _get_async_client(self): |
285 | return AsyncClient(self.username, self.password) | 285 | return AsyncClient(self.username, self.password) |
286 | |||
287 | |||
288 | class ClientPool(bb.asyncrpc.ClientPool): | ||
289 | def __init__( | ||
290 | self, | ||
291 | address, | ||
292 | max_clients, | ||
293 | *, | ||
294 | username=None, | ||
295 | password=None, | ||
296 | become=None, | ||
297 | ): | ||
298 | super().__init__(max_clients) | ||
299 | self.address = address | ||
300 | self.username = username | ||
301 | self.password = password | ||
302 | self.become = become | ||
303 | |||
304 | async def _new_client(self): | ||
305 | client = await create_async_client( | ||
306 | self.address, | ||
307 | username=self.username, | ||
308 | password=self.password, | ||
309 | ) | ||
310 | if self.become: | ||
311 | await client.become_user(self.become) | ||
312 | return client | ||
313 | |||
314 | def _run_key_tasks(self, queries, call): | ||
315 | results = {key: None for key in queries.keys()} | ||
316 | |||
317 | def make_task(key, args): | ||
318 | async def task(client): | ||
319 | nonlocal results | ||
320 | unihash = await call(client, args) | ||
321 | results[key] = unihash | ||
322 | |||
323 | return task | ||
324 | |||
325 | def gen_tasks(): | ||
326 | for key, args in queries.items(): | ||
327 | yield make_task(key, args) | ||
328 | |||
329 | self.run_tasks(gen_tasks()) | ||
330 | return results | ||
331 | |||
332 | def get_unihashes(self, queries): | ||
333 | """ | ||
334 | Query multiple unihashes in parallel. | ||
335 | |||
336 | The queries argument is a dictionary with arbitrary key. The values | ||
337 | must be a tuple of (method, taskhash). | ||
338 | |||
339 | Returns a dictionary with a corresponding key for each input key, and | ||
340 | the value is the queried unihash (which might be none if the query | ||
341 | failed) | ||
342 | """ | ||
343 | |||
344 | async def call(client, args): | ||
345 | method, taskhash = args | ||
346 | return await client.get_unihash(method, taskhash) | ||
347 | |||
348 | return self._run_key_tasks(queries, call) | ||
349 | |||
350 | def unihashes_exist(self, queries): | ||
351 | """ | ||
352 | Query multiple unihash existence checks in parallel. | ||
353 | |||
354 | The queries argument is a dictionary with arbitrary key. The values | ||
355 | must be a unihash. | ||
356 | |||
357 | Returns a dictionary with a corresponding key for each input key, and | ||
358 | the value is True or False if the unihash is known by the server (or | ||
359 | None if there was a failure) | ||
360 | """ | ||
361 | |||
362 | async def call(client, unihash): | ||
363 | return await client.unihash_exists(unihash) | ||
364 | |||
365 | return self._run_key_tasks(queries, call) | ||
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index fbbe81512a..0809453cf8 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -8,6 +8,7 @@ | |||
8 | from . import create_server, create_client | 8 | from . import create_server, create_client |
9 | from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS | 9 | from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS |
10 | from bb.asyncrpc import InvokeError | 10 | from bb.asyncrpc import InvokeError |
11 | from .client import ClientPool | ||
11 | import hashlib | 12 | import hashlib |
12 | import logging | 13 | import logging |
13 | import multiprocessing | 14 | import multiprocessing |
@@ -554,6 +555,88 @@ class HashEquivalenceCommonTests(object): | |||
554 | # shares a taskhash with Task 2 | 555 | # shares a taskhash with Task 2 |
555 | self.assertClientGetHash(self.client, taskhash2, unihash2) | 556 | self.assertClientGetHash(self.client, taskhash2, unihash2) |
556 | 557 | ||
558 | |||
559 | def test_client_pool_get_unihashes(self): | ||
560 | TEST_INPUT = ( | ||
561 | # taskhash outhash unihash | ||
562 | ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'), | ||
563 | # Duplicated taskhash with multiple output hashes and unihashes. | ||
564 | ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'), | ||
565 | # Equivalent hash | ||
566 | ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"), | ||
567 | ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"), | ||
568 | ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'), | ||
569 | ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'), | ||
570 | ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'), | ||
571 | ) | ||
572 | EXTRA_QUERIES = ( | ||
573 | "6b6be7a84ab179b4240c4302518dc3f6", | ||
574 | ) | ||
575 | |||
576 | with ClientPool(self.server_address, 10) as client_pool: | ||
577 | for taskhash, outhash, unihash in TEST_INPUT: | ||
578 | self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | ||
579 | |||
580 | query = {idx: (self.METHOD, data[0]) for idx, data in enumerate(TEST_INPUT)} | ||
581 | for idx, taskhash in enumerate(EXTRA_QUERIES): | ||
582 | query[idx + len(TEST_INPUT)] = (self.METHOD, taskhash) | ||
583 | |||
584 | result = client_pool.get_unihashes(query) | ||
585 | |||
586 | self.assertDictEqual(result, { | ||
587 | 0: "218e57509998197d570e2c98512d0105985dffc9", | ||
588 | 1: "218e57509998197d570e2c98512d0105985dffc9", | ||
589 | 2: "218e57509998197d570e2c98512d0105985dffc9", | ||
590 | 3: "3b5d3d83f07f259e9086fcb422c855286e18a57d", | ||
591 | 4: "f46d3fbb439bd9b921095da657a4de906510d2cd", | ||
592 | 5: "f46d3fbb439bd9b921095da657a4de906510d2cd", | ||
593 | 6: "05d2a63c81e32f0a36542ca677e8ad852365c538", | ||
594 | 7: None, | ||
595 | }) | ||
596 | |||
597 | def test_client_pool_unihash_exists(self): | ||
598 | TEST_INPUT = ( | ||
599 | # taskhash outhash unihash | ||
600 | ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'), | ||
601 | # Duplicated taskhash with multiple output hashes and unihashes. | ||
602 | ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'), | ||
603 | # Equivalent hash | ||
604 | ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"), | ||
605 | ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"), | ||
606 | ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'), | ||
607 | ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'), | ||
608 | ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'), | ||
609 | ) | ||
610 | EXTRA_QUERIES = ( | ||
611 | "6b6be7a84ab179b4240c4302518dc3f6", | ||
612 | ) | ||
613 | |||
614 | result_unihashes = set() | ||
615 | |||
616 | |||
617 | with ClientPool(self.server_address, 10) as client_pool: | ||
618 | for taskhash, outhash, unihash in TEST_INPUT: | ||
619 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | ||
620 | result_unihashes.add(result["unihash"]) | ||
621 | |||
622 | query = {} | ||
623 | expected = {} | ||
624 | |||
625 | for _, _, unihash in TEST_INPUT: | ||
626 | idx = len(query) | ||
627 | query[idx] = unihash | ||
628 | expected[idx] = unihash in result_unihashes | ||
629 | |||
630 | |||
631 | for unihash in EXTRA_QUERIES: | ||
632 | idx = len(query) | ||
633 | query[idx] = unihash | ||
634 | expected[idx] = False | ||
635 | |||
636 | result = client_pool.unihashes_exist(query) | ||
637 | self.assertDictEqual(result, expected) | ||
638 | |||
639 | |||
557 | def test_auth_read_perms(self): | 640 | def test_auth_read_perms(self): |
558 | admin_client = self.start_auth_server() | 641 | admin_client = self.start_auth_server() |
559 | 642 | ||