diff options
| -rw-r--r-- | bitbake/lib/bb/siggen.py | 53 |
1 files changed, 15 insertions, 38 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 65ca0811d5..79f347db30 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
| @@ -540,7 +540,7 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 540 | def __init__(self, data): | 540 | def __init__(self, data): |
| 541 | self.extramethod = {} | 541 | self.extramethod = {} |
| 542 | # NOTE: The cache only tracks hashes that exist. Hashes that don't | 542 | # NOTE: The cache only tracks hashes that exist. Hashes that don't |
| 543 | # exist are always queries from the server since it is possible for | 543 | # exist are always queried from the server since it is possible for |
| 544 | # hashes to appear over time, but much less likely for them to | 544 | # hashes to appear over time, but much less likely for them to |
| 545 | # disappear | 545 | # disappear |
| 546 | self.unihash_exists_cache = set() | 546 | self.unihash_exists_cache = set() |
| @@ -558,11 +558,11 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 558 | super().__init__(data) | 558 | super().__init__(data) |
| 559 | 559 | ||
| 560 | def get_taskdata(self): | 560 | def get_taskdata(self): |
| 561 | return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata() | 561 | return (self.server, self.method, self.extramethod, self.username, self.password, self.env) + super().get_taskdata() |
| 562 | 562 | ||
| 563 | def set_taskdata(self, data): | 563 | def set_taskdata(self, data): |
| 564 | self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7] | 564 | self.server, self.method, self.extramethod, self.username, self.password, self.env = data[:6] |
| 565 | super().set_taskdata(data[7:]) | 565 | super().set_taskdata(data[6:]) |
| 566 | 566 | ||
| 567 | def get_hashserv_creds(self): | 567 | def get_hashserv_creds(self): |
| 568 | if self.username and self.password: | 568 | if self.username and self.password: |
| @@ -595,13 +595,6 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 595 | self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) | 595 | self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) |
| 596 | yield self._client | 596 | yield self._client |
| 597 | 597 | ||
| 598 | @contextmanager | ||
| 599 | def client_pool(self): | ||
| 600 | with self._client_env(): | ||
| 601 | if getattr(self, '_client_pool', None) is None: | ||
| 602 | self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds()) | ||
| 603 | yield self._client_pool | ||
| 604 | |||
| 605 | def reset(self, data): | 598 | def reset(self, data): |
| 606 | self.__close_clients() | 599 | self.__close_clients() |
| 607 | return super().reset(data) | 600 | return super().reset(data) |
| @@ -686,15 +679,10 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 686 | else: | 679 | else: |
| 687 | uncached_query[key] = unihash | 680 | uncached_query[key] = unihash |
| 688 | 681 | ||
| 689 | if self.max_parallel <= 1 or len(uncached_query) <= 1: | 682 | with self.client() as client: |
| 690 | # No parallelism required. Make the query serially with the single client | 683 | uncached_result = { |
| 691 | with self.client() as client: | 684 | key: client.unihash_exists(value) for key, value in uncached_query.items() |
| 692 | uncached_result = { | 685 | } |
| 693 | key: client.unihash_exists(value) for key, value in uncached_query.items() | ||
| 694 | } | ||
| 695 | else: | ||
| 696 | with self.client_pool() as client_pool: | ||
| 697 | uncached_result = client_pool.unihashes_exist(uncached_query) | ||
| 698 | 686 | ||
| 699 | for key, exists in uncached_result.items(): | 687 | for key, exists in uncached_result.items(): |
| 700 | if exists: | 688 | if exists: |
| @@ -712,32 +700,20 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 712 | unihash | 700 | unihash |
| 713 | """ | 701 | """ |
| 714 | result = {} | 702 | result = {} |
| 715 | queries = {} | 703 | query_tids = [] |
| 716 | query_result = {} | ||
| 717 | 704 | ||
| 718 | for tid in tids: | 705 | for tid in tids: |
| 719 | unihash = self.get_cached_unihash(tid) | 706 | unihash = self.get_cached_unihash(tid) |
| 720 | if unihash: | 707 | if unihash: |
| 721 | result[tid] = unihash | 708 | result[tid] = unihash |
| 722 | else: | 709 | else: |
| 723 | queries[tid] = (self._get_method(tid), self.taskhash[tid]) | 710 | query_tids.append(tid) |
| 724 | |||
| 725 | if len(queries) == 0: | ||
| 726 | return result | ||
| 727 | 711 | ||
| 728 | if self.max_parallel <= 1 or len(queries) <= 1: | 712 | if query_tids: |
| 729 | # No parallelism required. Make the query using a single client | ||
| 730 | with self.client() as client: | 713 | with self.client() as client: |
| 731 | keys = list(queries.keys()) | 714 | unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids) |
| 732 | unihashes = client.get_unihash_batch(queries[k] for k in keys) | ||
| 733 | 715 | ||
| 734 | for idx, k in enumerate(keys): | 716 | for idx, tid in enumerate(query_tids): |
| 735 | query_result[k] = unihashes[idx] | ||
| 736 | else: | ||
| 737 | with self.client_pool() as client_pool: | ||
| 738 | query_result = client_pool.get_unihashes(queries) | ||
| 739 | |||
| 740 | for tid, unihash in query_result.items(): | ||
| 741 | # In the absence of being able to discover a unique hash from the | 717 | # In the absence of being able to discover a unique hash from the |
| 742 | # server, make it be equivalent to the taskhash. The unique "hash" only | 718 | # server, make it be equivalent to the taskhash. The unique "hash" only |
| 743 | # really needs to be a unique string (not even necessarily a hash), but | 719 | # really needs to be a unique string (not even necessarily a hash), but |
| @@ -752,6 +728,8 @@ class SignatureGeneratorUniHashMixIn(object): | |||
| 752 | # to the server, there is a better chance that they will agree on | 728 | # to the server, there is a better chance that they will agree on |
| 753 | # the unique hash. | 729 | # the unique hash. |
| 754 | taskhash = self.taskhash[tid] | 730 | taskhash = self.taskhash[tid] |
| 731 | unihash = unihashes[idx] | ||
| 732 | |||
| 755 | if unihash: | 733 | if unihash: |
| 756 | # A unique hash equal to the taskhash is not very interesting, | 734 | # A unique hash equal to the taskhash is not very interesting, |
| 757 | # so it is reported it at debug level 2. If they differ, that | 735 | # so it is reported it at debug level 2. If they differ, that |
| @@ -898,7 +876,6 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG | |||
| 898 | super().init_rundepcheck(data) | 876 | super().init_rundepcheck(data) |
| 899 | self.server = data.getVar('BB_HASHSERVE') | 877 | self.server = data.getVar('BB_HASHSERVE') |
| 900 | self.method = "sstate_output_hash" | 878 | self.method = "sstate_output_hash" |
| 901 | self.max_parallel = 1 | ||
| 902 | 879 | ||
| 903 | def clean_checksum_file_path(file_checksum_tuple): | 880 | def clean_checksum_file_path(file_checksum_tuple): |
| 904 | f, cs = file_checksum_tuple | 881 | f, cs = file_checksum_tuple |
