diff options
Diffstat (limited to 'bitbake/bin/bitbake-hashclient')
-rwxr-xr-x | bitbake/bin/bitbake-hashclient | 246 |
1 files changed, 226 insertions, 20 deletions
diff --git a/bitbake/bin/bitbake-hashclient b/bitbake/bin/bitbake-hashclient index a89290217b..610787ed2b 100755 --- a/bitbake/bin/bitbake-hashclient +++ b/bitbake/bin/bitbake-hashclient | |||
@@ -13,6 +13,10 @@ import pprint | |||
13 | import sys | 13 | import sys |
14 | import threading | 14 | import threading |
15 | import time | 15 | import time |
16 | import warnings | ||
17 | import netrc | ||
18 | import json | ||
19 | warnings.simplefilter("default") | ||
16 | 20 | ||
17 | try: | 21 | try: |
18 | import tqdm | 22 | import tqdm |
@@ -34,18 +38,42 @@ except ImportError: | |||
34 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) | 38 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) |
35 | 39 | ||
36 | import hashserv | 40 | import hashserv |
41 | import bb.asyncrpc | ||
37 | 42 | ||
38 | DEFAULT_ADDRESS = 'unix://./hashserve.sock' | 43 | DEFAULT_ADDRESS = 'unix://./hashserve.sock' |
39 | METHOD = 'stress.test.method' | 44 | METHOD = 'stress.test.method' |
40 | 45 | ||
46 | def print_user(u): | ||
47 | print(f"Username: {u['username']}") | ||
48 | if "permissions" in u: | ||
49 | print("Permissions: " + " ".join(u["permissions"])) | ||
50 | if "token" in u: | ||
51 | print(f"Token: {u['token']}") | ||
52 | |||
41 | 53 | ||
42 | def main(): | 54 | def main(): |
55 | def handle_get(args, client): | ||
56 | result = client.get_taskhash(args.method, args.taskhash, all_properties=True) | ||
57 | if not result: | ||
58 | return 0 | ||
59 | |||
60 | print(json.dumps(result, sort_keys=True, indent=4)) | ||
61 | return 0 | ||
62 | |||
63 | def handle_get_outhash(args, client): | ||
64 | result = client.get_outhash(args.method, args.outhash, args.taskhash) | ||
65 | if not result: | ||
66 | return 0 | ||
67 | |||
68 | print(json.dumps(result, sort_keys=True, indent=4)) | ||
69 | return 0 | ||
70 | |||
43 | def handle_stats(args, client): | 71 | def handle_stats(args, client): |
44 | if args.reset: | 72 | if args.reset: |
45 | s = client.reset_stats() | 73 | s = client.reset_stats() |
46 | else: | 74 | else: |
47 | s = client.get_stats() | 75 | s = client.get_stats() |
48 | pprint.pprint(s) | 76 | print(json.dumps(s, sort_keys=True, indent=4)) |
49 | return 0 | 77 | return 0 |
50 | 78 | ||
51 | def handle_stress(args, client): | 79 | def handle_stress(args, client): |
@@ -54,25 +82,24 @@ def main(): | |||
54 | nonlocal missed_hashes | 82 | nonlocal missed_hashes |
55 | nonlocal max_time | 83 | nonlocal max_time |
56 | 84 | ||
57 | client = hashserv.create_client(args.address) | 85 | with hashserv.create_client(args.address) as client: |
58 | 86 | for i in range(args.requests): | |
59 | for i in range(args.requests): | 87 | taskhash = hashlib.sha256() |
60 | taskhash = hashlib.sha256() | 88 | taskhash.update(args.taskhash_seed.encode('utf-8')) |
61 | taskhash.update(args.taskhash_seed.encode('utf-8')) | 89 | taskhash.update(str(i).encode('utf-8')) |
62 | taskhash.update(str(i).encode('utf-8')) | ||
63 | 90 | ||
64 | start_time = time.perf_counter() | 91 | start_time = time.perf_counter() |
65 | l = client.get_unihash(METHOD, taskhash.hexdigest()) | 92 | l = client.get_unihash(METHOD, taskhash.hexdigest()) |
66 | elapsed = time.perf_counter() - start_time | 93 | elapsed = time.perf_counter() - start_time |
67 | 94 | ||
68 | with lock: | 95 | with lock: |
69 | if l: | 96 | if l: |
70 | found_hashes += 1 | 97 | found_hashes += 1 |
71 | else: | 98 | else: |
72 | missed_hashes += 1 | 99 | missed_hashes += 1 |
73 | 100 | ||
74 | max_time = max(elapsed, max_time) | 101 | max_time = max(elapsed, max_time) |
75 | pbar.update() | 102 | pbar.update() |
76 | 103 | ||
77 | max_time = 0 | 104 | max_time = 0 |
78 | found_hashes = 0 | 105 | found_hashes = 0 |
@@ -111,12 +138,114 @@ def main(): | |||
111 | with lock: | 138 | with lock: |
112 | pbar.update() | 139 | pbar.update() |
113 | 140 | ||
141 | def handle_remove(args, client): | ||
142 | where = {k: v for k, v in args.where} | ||
143 | if where: | ||
144 | result = client.remove(where) | ||
145 | print("Removed %d row(s)" % (result["count"])) | ||
146 | else: | ||
147 | print("No query specified") | ||
148 | |||
149 | def handle_clean_unused(args, client): | ||
150 | result = client.clean_unused(args.max_age) | ||
151 | print("Removed %d rows" % (result["count"])) | ||
152 | return 0 | ||
153 | |||
154 | def handle_refresh_token(args, client): | ||
155 | r = client.refresh_token(args.username) | ||
156 | print_user(r) | ||
157 | |||
158 | def handle_set_user_permissions(args, client): | ||
159 | r = client.set_user_perms(args.username, args.permissions) | ||
160 | print_user(r) | ||
161 | |||
162 | def handle_get_user(args, client): | ||
163 | r = client.get_user(args.username) | ||
164 | print_user(r) | ||
165 | |||
166 | def handle_get_all_users(args, client): | ||
167 | users = client.get_all_users() | ||
168 | print("{username:20}| {permissions}".format(username="Username", permissions="Permissions")) | ||
169 | print(("-" * 20) + "+" + ("-" * 20)) | ||
170 | for u in users: | ||
171 | print("{username:20}| {permissions}".format(username=u["username"], permissions=" ".join(u["permissions"]))) | ||
172 | |||
173 | def handle_new_user(args, client): | ||
174 | r = client.new_user(args.username, args.permissions) | ||
175 | print_user(r) | ||
176 | |||
177 | def handle_delete_user(args, client): | ||
178 | r = client.delete_user(args.username) | ||
179 | print_user(r) | ||
180 | |||
181 | def handle_get_db_usage(args, client): | ||
182 | usage = client.get_db_usage() | ||
183 | print(usage) | ||
184 | tables = sorted(usage.keys()) | ||
185 | print("{name:20}| {rows:20}".format(name="Table name", rows="Rows")) | ||
186 | print(("-" * 20) + "+" + ("-" * 20)) | ||
187 | for t in tables: | ||
188 | print("{name:20}| {rows:<20}".format(name=t, rows=usage[t]["rows"])) | ||
189 | print() | ||
190 | |||
191 | total_rows = sum(t["rows"] for t in usage.values()) | ||
192 | print(f"Total rows: {total_rows}") | ||
193 | |||
194 | def handle_get_db_query_columns(args, client): | ||
195 | columns = client.get_db_query_columns() | ||
196 | print("\n".join(sorted(columns))) | ||
197 | |||
198 | def handle_gc_status(args, client): | ||
199 | result = client.gc_status() | ||
200 | if not result["mark"]: | ||
201 | print("No Garbage collection in progress") | ||
202 | return 0 | ||
203 | |||
204 | print("Current Mark: %s" % result["mark"]) | ||
205 | print("Total hashes to keep: %d" % result["keep"]) | ||
206 | print("Total hashes to remove: %s" % result["remove"]) | ||
207 | return 0 | ||
208 | |||
209 | def handle_gc_mark(args, client): | ||
210 | where = {k: v for k, v in args.where} | ||
211 | result = client.gc_mark(args.mark, where) | ||
212 | print("New hashes marked: %d" % result["count"]) | ||
213 | return 0 | ||
214 | |||
215 | def handle_gc_sweep(args, client): | ||
216 | result = client.gc_sweep(args.mark) | ||
217 | print("Removed %d rows" % result["count"]) | ||
218 | return 0 | ||
219 | |||
220 | def handle_unihash_exists(args, client): | ||
221 | result = client.unihash_exists(args.unihash) | ||
222 | if args.quiet: | ||
223 | return 0 if result else 1 | ||
224 | |||
225 | print("true" if result else "false") | ||
226 | return 0 | ||
227 | |||
114 | parser = argparse.ArgumentParser(description='Hash Equivalence Client') | 228 | parser = argparse.ArgumentParser(description='Hash Equivalence Client') |
115 | parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")') | 229 | parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")') |
116 | parser.add_argument('--log', default='WARNING', help='Set logging level') | 230 | parser.add_argument('--log', default='WARNING', help='Set logging level') |
231 | parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME") | ||
232 | parser.add_argument('--password', '-p', metavar="TOKEN", help="Authenticate using token TOKEN") | ||
233 | parser.add_argument('--become', '-b', metavar="USERNAME", help="Impersonate user USERNAME (if allowed) when performing actions") | ||
234 | parser.add_argument('--no-netrc', '-n', action="store_false", dest="netrc", help="Do not use .netrc") | ||
117 | 235 | ||
118 | subparsers = parser.add_subparsers() | 236 | subparsers = parser.add_subparsers() |
119 | 237 | ||
238 | get_parser = subparsers.add_parser('get', help="Get the unihash for a taskhash") | ||
239 | get_parser.add_argument("method", help="Method to query") | ||
240 | get_parser.add_argument("taskhash", help="Task hash to query") | ||
241 | get_parser.set_defaults(func=handle_get) | ||
242 | |||
243 | get_outhash_parser = subparsers.add_parser('get-outhash', help="Get output hash information") | ||
244 | get_outhash_parser.add_argument("method", help="Method to query") | ||
245 | get_outhash_parser.add_argument("outhash", help="Output hash to query") | ||
246 | get_outhash_parser.add_argument("taskhash", help="Task hash to query") | ||
247 | get_outhash_parser.set_defaults(func=handle_get_outhash) | ||
248 | |||
120 | stats_parser = subparsers.add_parser('stats', help='Show server stats') | 249 | stats_parser = subparsers.add_parser('stats', help='Show server stats') |
121 | stats_parser.add_argument('--reset', action='store_true', | 250 | stats_parser.add_argument('--reset', action='store_true', |
122 | help='Reset server stats') | 251 | help='Reset server stats') |
@@ -135,6 +264,64 @@ def main(): | |||
135 | help='Include string in outhash') | 264 | help='Include string in outhash') |
136 | stress_parser.set_defaults(func=handle_stress) | 265 | stress_parser.set_defaults(func=handle_stress) |
137 | 266 | ||
267 | remove_parser = subparsers.add_parser('remove', help="Remove hash entries") | ||
268 | remove_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[], | ||
269 | help="Remove entries from table where KEY == VALUE") | ||
270 | remove_parser.set_defaults(func=handle_remove) | ||
271 | |||
272 | clean_unused_parser = subparsers.add_parser('clean-unused', help="Remove unused database entries") | ||
273 | clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old") | ||
274 | clean_unused_parser.set_defaults(func=handle_clean_unused) | ||
275 | |||
276 | refresh_token_parser = subparsers.add_parser('refresh-token', help="Refresh auth token") | ||
277 | refresh_token_parser.add_argument("--username", "-u", help="Refresh the token for another user (if authorized)") | ||
278 | refresh_token_parser.set_defaults(func=handle_refresh_token) | ||
279 | |||
280 | set_user_perms_parser = subparsers.add_parser('set-user-perms', help="Set new permissions for user") | ||
281 | set_user_perms_parser.add_argument("--username", "-u", help="Username", required=True) | ||
282 | set_user_perms_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions") | ||
283 | set_user_perms_parser.set_defaults(func=handle_set_user_permissions) | ||
284 | |||
285 | get_user_parser = subparsers.add_parser('get-user', help="Get user") | ||
286 | get_user_parser.add_argument("--username", "-u", help="Username") | ||
287 | get_user_parser.set_defaults(func=handle_get_user) | ||
288 | |||
289 | get_all_users_parser = subparsers.add_parser('get-all-users', help="List all users") | ||
290 | get_all_users_parser.set_defaults(func=handle_get_all_users) | ||
291 | |||
292 | new_user_parser = subparsers.add_parser('new-user', help="Create new user") | ||
293 | new_user_parser.add_argument("--username", "-u", help="Username", required=True) | ||
294 | new_user_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions") | ||
295 | new_user_parser.set_defaults(func=handle_new_user) | ||
296 | |||
297 | delete_user_parser = subparsers.add_parser('delete-user', help="Delete user") | ||
298 | delete_user_parser.add_argument("--username", "-u", help="Username", required=True) | ||
299 | delete_user_parser.set_defaults(func=handle_delete_user) | ||
300 | |||
301 | db_usage_parser = subparsers.add_parser('get-db-usage', help="Database Usage") | ||
302 | db_usage_parser.set_defaults(func=handle_get_db_usage) | ||
303 | |||
304 | db_query_columns_parser = subparsers.add_parser('get-db-query-columns', help="Show columns that can be used in database queries") | ||
305 | db_query_columns_parser.set_defaults(func=handle_get_db_query_columns) | ||
306 | |||
307 | gc_status_parser = subparsers.add_parser("gc-status", help="Show garbage collection status") | ||
308 | gc_status_parser.set_defaults(func=handle_gc_status) | ||
309 | |||
310 | gc_mark_parser = subparsers.add_parser('gc-mark', help="Mark hashes to be kept for garbage collection") | ||
311 | gc_mark_parser.add_argument("mark", help="Mark for this garbage collection operation") | ||
312 | gc_mark_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[], | ||
313 | help="Keep entries in table where KEY == VALUE") | ||
314 | gc_mark_parser.set_defaults(func=handle_gc_mark) | ||
315 | |||
316 | gc_sweep_parser = subparsers.add_parser('gc-sweep', help="Perform garbage collection and delete any entries that are not marked") | ||
317 | gc_sweep_parser.add_argument("mark", help="Mark for this garbage collection operation") | ||
318 | gc_sweep_parser.set_defaults(func=handle_gc_sweep) | ||
319 | |||
320 | unihash_exists_parser = subparsers.add_parser('unihash-exists', help="Check if a unihash is known to the server") | ||
321 | unihash_exists_parser.add_argument("--quiet", action="store_true", help="Don't print status. Instead, exit with 0 if unihash exists and 1 if it does not") | ||
322 | unihash_exists_parser.add_argument("unihash", help="Unihash to check") | ||
323 | unihash_exists_parser.set_defaults(func=handle_unihash_exists) | ||
324 | |||
138 | args = parser.parse_args() | 325 | args = parser.parse_args() |
139 | 326 | ||
140 | logger = logging.getLogger('hashserv') | 327 | logger = logging.getLogger('hashserv') |
@@ -148,11 +335,30 @@ def main(): | |||
148 | console.setLevel(level) | 335 | console.setLevel(level) |
149 | logger.addHandler(console) | 336 | logger.addHandler(console) |
150 | 337 | ||
338 | login = args.login | ||
339 | password = args.password | ||
340 | |||
341 | if login is None and args.netrc: | ||
342 | try: | ||
343 | n = netrc.netrc() | ||
344 | auth = n.authenticators(args.address) | ||
345 | if auth is not None: | ||
346 | login, _, password = auth | ||
347 | except FileNotFoundError: | ||
348 | pass | ||
349 | except netrc.NetrcParseError as e: | ||
350 | sys.stderr.write(f"Error parsing {e.filename}:{e.lineno}: {e.msg}\n") | ||
351 | |||
151 | func = getattr(args, 'func', None) | 352 | func = getattr(args, 'func', None) |
152 | if func: | 353 | if func: |
153 | client = hashserv.create_client(args.address) | 354 | try: |
154 | 355 | with hashserv.create_client(args.address, login, password) as client: | |
155 | return func(args, client) | 356 | if args.become: |
357 | client.become_user(args.become) | ||
358 | return func(args, client) | ||
359 | except bb.asyncrpc.InvokeError as e: | ||
360 | print(f"ERROR: {e}") | ||
361 | return 1 | ||
156 | 362 | ||
157 | return 0 | 363 | return 0 |
158 | 364 | ||