diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2020-06-25 09:21:07 -0500 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-07-02 16:11:40 +0100 |
commit | 6ebf01bfd43b6d95a70699b1e58a42fd7d1002a6 (patch) | |
tree | 3f18afa2f1918dde70ced1013e5d859cc4c573e7 /bitbake/lib/hashserv/client.py | |
parent | b6e0f5889eb55d88276807407f75eaad9bf0a96a (diff) | |
download | poky-6ebf01bfd43b6d95a70699b1e58a42fd7d1002a6.tar.gz |
bitbake: hashserv: Chunkify large messages
The hash equivalence client and server can occasionally send messages
that are too large for the server to fit in the receive buffer (64 KB).
To prevent this, support is added to the protocol to "chunkify" the
stream and break it up into manageable pieces that the server can each
side can back together.
Ideally, this would be negotiated by the client and server, but it's
currently hard coded to 32 KB to prevent the round-trip delay.
(Bitbake rev: 1a7bddb5471a02a744e7a441a3b4a6da693348b0)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit e27a28c1e40e886ee68ba4b99b537ffc9c3577d4)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/hashserv/client.py')
-rw-r--r-- | bitbake/lib/hashserv/client.py | 43 |
1 files changed, 35 insertions, 8 deletions
diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index 46085d6418..a29af836d9 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py | |||
@@ -7,6 +7,7 @@ import json | |||
7 | import logging | 7 | import logging |
8 | import socket | 8 | import socket |
9 | import os | 9 | import os |
10 | from . import chunkify, DEFAULT_MAX_CHUNK | ||
10 | 11 | ||
11 | 12 | ||
12 | logger = logging.getLogger('hashserv.client') | 13 | logger = logging.getLogger('hashserv.client') |
@@ -25,6 +26,7 @@ class Client(object): | |||
25 | self.reader = None | 26 | self.reader = None |
26 | self.writer = None | 27 | self.writer = None |
27 | self.mode = self.MODE_NORMAL | 28 | self.mode = self.MODE_NORMAL |
29 | self.max_chunk = DEFAULT_MAX_CHUNK | ||
28 | 30 | ||
29 | def connect_tcp(self, address, port): | 31 | def connect_tcp(self, address, port): |
30 | def connect_sock(): | 32 | def connect_sock(): |
@@ -58,7 +60,7 @@ class Client(object): | |||
58 | self.reader = self._socket.makefile('r', encoding='utf-8') | 60 | self.reader = self._socket.makefile('r', encoding='utf-8') |
59 | self.writer = self._socket.makefile('w', encoding='utf-8') | 61 | self.writer = self._socket.makefile('w', encoding='utf-8') |
60 | 62 | ||
61 | self.writer.write('OEHASHEQUIV 1.0\n\n') | 63 | self.writer.write('OEHASHEQUIV 1.1\n\n') |
62 | self.writer.flush() | 64 | self.writer.flush() |
63 | 65 | ||
64 | # Restore mode if the socket is being re-created | 66 | # Restore mode if the socket is being re-created |
@@ -91,18 +93,35 @@ class Client(object): | |||
91 | count += 1 | 93 | count += 1 |
92 | 94 | ||
93 | def send_message(self, msg): | 95 | def send_message(self, msg): |
96 | def get_line(): | ||
97 | line = self.reader.readline() | ||
98 | if not line: | ||
99 | raise HashConnectionError('Connection closed') | ||
100 | |||
101 | if not line.endswith('\n'): | ||
102 | raise HashConnectionError('Bad message %r' % message) | ||
103 | |||
104 | return line | ||
105 | |||
94 | def proc(): | 106 | def proc(): |
95 | self.writer.write('%s\n' % json.dumps(msg)) | 107 | for c in chunkify(json.dumps(msg), self.max_chunk): |
108 | self.writer.write(c) | ||
96 | self.writer.flush() | 109 | self.writer.flush() |
97 | 110 | ||
98 | l = self.reader.readline() | 111 | l = get_line() |
99 | if not l: | ||
100 | raise HashConnectionError('Connection closed') | ||
101 | 112 | ||
102 | if not l.endswith('\n'): | 113 | m = json.loads(l) |
103 | raise HashConnectionError('Bad message %r' % message) | 114 | if 'chunk-stream' in m: |
115 | lines = [] | ||
116 | while True: | ||
117 | l = get_line().rstrip('\n') | ||
118 | if not l: | ||
119 | break | ||
120 | lines.append(l) | ||
104 | 121 | ||
105 | return json.loads(l) | 122 | m = json.loads(''.join(lines)) |
123 | |||
124 | return m | ||
106 | 125 | ||
107 | return self._send_wrapper(proc) | 126 | return self._send_wrapper(proc) |
108 | 127 | ||
@@ -155,6 +174,14 @@ class Client(object): | |||
155 | m['unihash'] = unihash | 174 | m['unihash'] = unihash |
156 | return self.send_message({'report-equiv': m}) | 175 | return self.send_message({'report-equiv': m}) |
157 | 176 | ||
177 | def get_taskhash(self, method, taskhash, all_properties=False): | ||
178 | self._set_mode(self.MODE_NORMAL) | ||
179 | return self.send_message({'get': { | ||
180 | 'taskhash': taskhash, | ||
181 | 'method': method, | ||
182 | 'all': all_properties | ||
183 | }}) | ||
184 | |||
158 | def get_stats(self): | 185 | def get_stats(self): |
159 | self._set_mode(self.MODE_NORMAL) | 186 | self._set_mode(self.MODE_NORMAL) |
160 | return self.send_message({'get-stats': None}) | 187 | return self.send_message({'get-stats': None}) |