diff options
Diffstat (limited to 'bitbake/lib/hashserv')
-rw-r--r-- | bitbake/lib/hashserv/sqlalchemy.py | 297 |
1 files changed, 140 insertions, 157 deletions
diff --git a/bitbake/lib/hashserv/sqlalchemy.py b/bitbake/lib/hashserv/sqlalchemy.py index 89a6b86d9d..873547809a 100644 --- a/bitbake/lib/hashserv/sqlalchemy.py +++ b/bitbake/lib/hashserv/sqlalchemy.py | |||
@@ -233,124 +233,113 @@ class Database(object): | |||
233 | return row.value | 233 | return row.value |
234 | 234 | ||
235 | async def get_unihash_by_taskhash_full(self, method, taskhash): | 235 | async def get_unihash_by_taskhash_full(self, method, taskhash): |
236 | statement = ( | ||
237 | select( | ||
238 | OuthashesV2, | ||
239 | UnihashesV3.unihash.label("unihash"), | ||
240 | ) | ||
241 | .join( | ||
242 | UnihashesV3, | ||
243 | and_( | ||
244 | UnihashesV3.method == OuthashesV2.method, | ||
245 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
246 | ), | ||
247 | ) | ||
248 | .where( | ||
249 | OuthashesV2.method == method, | ||
250 | OuthashesV2.taskhash == taskhash, | ||
251 | ) | ||
252 | .order_by( | ||
253 | OuthashesV2.created.asc(), | ||
254 | ) | ||
255 | .limit(1) | ||
256 | ) | ||
257 | self.logger.debug("%s", statement) | ||
258 | async with self.db.begin(): | 236 | async with self.db.begin(): |
259 | result = await self.db.execute(statement) | 237 | result = await self._execute( |
238 | select( | ||
239 | OuthashesV2, | ||
240 | UnihashesV3.unihash.label("unihash"), | ||
241 | ) | ||
242 | .join( | ||
243 | UnihashesV3, | ||
244 | and_( | ||
245 | UnihashesV3.method == OuthashesV2.method, | ||
246 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
247 | ), | ||
248 | ) | ||
249 | .where( | ||
250 | OuthashesV2.method == method, | ||
251 | OuthashesV2.taskhash == taskhash, | ||
252 | ) | ||
253 | .order_by( | ||
254 | OuthashesV2.created.asc(), | ||
255 | ) | ||
256 | .limit(1) | ||
257 | ) | ||
260 | return map_row(result.first()) | 258 | return map_row(result.first()) |
261 | 259 | ||
262 | async def get_unihash_by_outhash(self, method, outhash): | 260 | async def get_unihash_by_outhash(self, method, outhash): |
263 | statement = ( | ||
264 | select(OuthashesV2, UnihashesV3.unihash.label("unihash")) | ||
265 | .join( | ||
266 | UnihashesV3, | ||
267 | and_( | ||
268 | UnihashesV3.method == OuthashesV2.method, | ||
269 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
270 | ), | ||
271 | ) | ||
272 | .where( | ||
273 | OuthashesV2.method == method, | ||
274 | OuthashesV2.outhash == outhash, | ||
275 | ) | ||
276 | .order_by( | ||
277 | OuthashesV2.created.asc(), | ||
278 | ) | ||
279 | .limit(1) | ||
280 | ) | ||
281 | self.logger.debug("%s", statement) | ||
282 | async with self.db.begin(): | 261 | async with self.db.begin(): |
283 | result = await self.db.execute(statement) | 262 | result = await self._execute( |
263 | select(OuthashesV2, UnihashesV3.unihash.label("unihash")) | ||
264 | .join( | ||
265 | UnihashesV3, | ||
266 | and_( | ||
267 | UnihashesV3.method == OuthashesV2.method, | ||
268 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
269 | ), | ||
270 | ) | ||
271 | .where( | ||
272 | OuthashesV2.method == method, | ||
273 | OuthashesV2.outhash == outhash, | ||
274 | ) | ||
275 | .order_by( | ||
276 | OuthashesV2.created.asc(), | ||
277 | ) | ||
278 | .limit(1) | ||
279 | ) | ||
284 | return map_row(result.first()) | 280 | return map_row(result.first()) |
285 | 281 | ||
286 | async def get_outhash(self, method, outhash): | 282 | async def get_outhash(self, method, outhash): |
287 | statement = ( | ||
288 | select(OuthashesV2) | ||
289 | .where( | ||
290 | OuthashesV2.method == method, | ||
291 | OuthashesV2.outhash == outhash, | ||
292 | ) | ||
293 | .order_by( | ||
294 | OuthashesV2.created.asc(), | ||
295 | ) | ||
296 | .limit(1) | ||
297 | ) | ||
298 | |||
299 | self.logger.debug("%s", statement) | ||
300 | async with self.db.begin(): | 283 | async with self.db.begin(): |
301 | result = await self.db.execute(statement) | 284 | result = await self._execute( |
285 | select(OuthashesV2) | ||
286 | .where( | ||
287 | OuthashesV2.method == method, | ||
288 | OuthashesV2.outhash == outhash, | ||
289 | ) | ||
290 | .order_by( | ||
291 | OuthashesV2.created.asc(), | ||
292 | ) | ||
293 | .limit(1) | ||
294 | ) | ||
302 | return map_row(result.first()) | 295 | return map_row(result.first()) |
303 | 296 | ||
304 | async def get_equivalent_for_outhash(self, method, outhash, taskhash): | 297 | async def get_equivalent_for_outhash(self, method, outhash, taskhash): |
305 | statement = ( | ||
306 | select( | ||
307 | OuthashesV2.taskhash.label("taskhash"), | ||
308 | UnihashesV3.unihash.label("unihash"), | ||
309 | ) | ||
310 | .join( | ||
311 | UnihashesV3, | ||
312 | and_( | ||
313 | UnihashesV3.method == OuthashesV2.method, | ||
314 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
315 | ), | ||
316 | ) | ||
317 | .where( | ||
318 | OuthashesV2.method == method, | ||
319 | OuthashesV2.outhash == outhash, | ||
320 | OuthashesV2.taskhash != taskhash, | ||
321 | ) | ||
322 | .order_by( | ||
323 | OuthashesV2.created.asc(), | ||
324 | ) | ||
325 | .limit(1) | ||
326 | ) | ||
327 | self.logger.debug("%s", statement) | ||
328 | async with self.db.begin(): | 298 | async with self.db.begin(): |
329 | result = await self.db.execute(statement) | 299 | result = await self._execute( |
300 | select( | ||
301 | OuthashesV2.taskhash.label("taskhash"), | ||
302 | UnihashesV3.unihash.label("unihash"), | ||
303 | ) | ||
304 | .join( | ||
305 | UnihashesV3, | ||
306 | and_( | ||
307 | UnihashesV3.method == OuthashesV2.method, | ||
308 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
309 | ), | ||
310 | ) | ||
311 | .where( | ||
312 | OuthashesV2.method == method, | ||
313 | OuthashesV2.outhash == outhash, | ||
314 | OuthashesV2.taskhash != taskhash, | ||
315 | ) | ||
316 | .order_by( | ||
317 | OuthashesV2.created.asc(), | ||
318 | ) | ||
319 | .limit(1) | ||
320 | ) | ||
330 | return map_row(result.first()) | 321 | return map_row(result.first()) |
331 | 322 | ||
332 | async def get_equivalent(self, method, taskhash): | 323 | async def get_equivalent(self, method, taskhash): |
333 | statement = select( | ||
334 | UnihashesV3.unihash, | ||
335 | UnihashesV3.method, | ||
336 | UnihashesV3.taskhash, | ||
337 | ).where( | ||
338 | UnihashesV3.method == method, | ||
339 | UnihashesV3.taskhash == taskhash, | ||
340 | ) | ||
341 | self.logger.debug("%s", statement) | ||
342 | async with self.db.begin(): | 324 | async with self.db.begin(): |
343 | result = await self.db.execute(statement) | 325 | result = await self._execute( |
326 | select( | ||
327 | UnihashesV3.unihash, | ||
328 | UnihashesV3.method, | ||
329 | UnihashesV3.taskhash, | ||
330 | ).where( | ||
331 | UnihashesV3.method == method, | ||
332 | UnihashesV3.taskhash == taskhash, | ||
333 | ) | ||
334 | ) | ||
344 | return map_row(result.first()) | 335 | return map_row(result.first()) |
345 | 336 | ||
346 | async def remove(self, condition): | 337 | async def remove(self, condition): |
347 | async def do_remove(table): | 338 | async def do_remove(table): |
348 | where = _make_condition_statement(table, condition) | 339 | where = _make_condition_statement(table, condition) |
349 | if where: | 340 | if where: |
350 | statement = delete(table).where(*where) | ||
351 | self.logger.debug("%s", statement) | ||
352 | async with self.db.begin(): | 341 | async with self.db.begin(): |
353 | result = await self.db.execute(statement) | 342 | result = await self._execute(delete(table).where(*where)) |
354 | return result.rowcount | 343 | return result.rowcount |
355 | 344 | ||
356 | return 0 | 345 | return 0 |
@@ -417,21 +406,21 @@ class Database(object): | |||
417 | return result.rowcount | 406 | return result.rowcount |
418 | 407 | ||
419 | async def clean_unused(self, oldest): | 408 | async def clean_unused(self, oldest): |
420 | statement = delete(OuthashesV2).where( | ||
421 | OuthashesV2.created < oldest, | ||
422 | ~( | ||
423 | select(UnihashesV3.id) | ||
424 | .where( | ||
425 | UnihashesV3.method == OuthashesV2.method, | ||
426 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
427 | ) | ||
428 | .limit(1) | ||
429 | .exists() | ||
430 | ), | ||
431 | ) | ||
432 | self.logger.debug("%s", statement) | ||
433 | async with self.db.begin(): | 409 | async with self.db.begin(): |
434 | result = await self.db.execute(statement) | 410 | result = await self._execute( |
411 | delete(OuthashesV2).where( | ||
412 | OuthashesV2.created < oldest, | ||
413 | ~( | ||
414 | select(UnihashesV3.id) | ||
415 | .where( | ||
416 | UnihashesV3.method == OuthashesV2.method, | ||
417 | UnihashesV3.taskhash == OuthashesV2.taskhash, | ||
418 | ) | ||
419 | .limit(1) | ||
420 | .exists() | ||
421 | ), | ||
422 | ) | ||
423 | ) | ||
435 | return result.rowcount | 424 | return result.rowcount |
436 | 425 | ||
437 | async def insert_unihash(self, method, taskhash, unihash): | 426 | async def insert_unihash(self, method, taskhash, unihash): |
@@ -461,11 +450,9 @@ class Database(object): | |||
461 | if "created" in data and not isinstance(data["created"], datetime): | 450 | if "created" in data and not isinstance(data["created"], datetime): |
462 | data["created"] = datetime.fromisoformat(data["created"]) | 451 | data["created"] = datetime.fromisoformat(data["created"]) |
463 | 452 | ||
464 | statement = insert(OuthashesV2).values(**data) | ||
465 | self.logger.debug("%s", statement) | ||
466 | try: | 453 | try: |
467 | async with self.db.begin(): | 454 | async with self.db.begin(): |
468 | await self.db.execute(statement) | 455 | await self._execute(insert(OuthashesV2).values(**data)) |
469 | return True | 456 | return True |
470 | except IntegrityError: | 457 | except IntegrityError: |
471 | self.logger.debug( | 458 | self.logger.debug( |
@@ -474,16 +461,16 @@ class Database(object): | |||
474 | return False | 461 | return False |
475 | 462 | ||
476 | async def _get_user(self, username): | 463 | async def _get_user(self, username): |
477 | statement = select( | ||
478 | Users.username, | ||
479 | Users.permissions, | ||
480 | Users.token, | ||
481 | ).where( | ||
482 | Users.username == username, | ||
483 | ) | ||
484 | self.logger.debug("%s", statement) | ||
485 | async with self.db.begin(): | 464 | async with self.db.begin(): |
486 | result = await self.db.execute(statement) | 465 | result = await self._execute( |
466 | select( | ||
467 | Users.username, | ||
468 | Users.permissions, | ||
469 | Users.token, | ||
470 | ).where( | ||
471 | Users.username == username, | ||
472 | ) | ||
473 | ) | ||
487 | return result.first() | 474 | return result.first() |
488 | 475 | ||
489 | async def lookup_user_token(self, username): | 476 | async def lookup_user_token(self, username): |
@@ -496,70 +483,66 @@ class Database(object): | |||
496 | return map_user(await self._get_user(username)) | 483 | return map_user(await self._get_user(username)) |
497 | 484 | ||
498 | async def set_user_token(self, username, token): | 485 | async def set_user_token(self, username, token): |
499 | statement = ( | ||
500 | update(Users) | ||
501 | .where( | ||
502 | Users.username == username, | ||
503 | ) | ||
504 | .values( | ||
505 | token=token, | ||
506 | ) | ||
507 | ) | ||
508 | self.logger.debug("%s", statement) | ||
509 | async with self.db.begin(): | 486 | async with self.db.begin(): |
510 | result = await self.db.execute(statement) | 487 | result = await self._execute( |
488 | update(Users) | ||
489 | .where( | ||
490 | Users.username == username, | ||
491 | ) | ||
492 | .values( | ||
493 | token=token, | ||
494 | ) | ||
495 | ) | ||
511 | return result.rowcount != 0 | 496 | return result.rowcount != 0 |
512 | 497 | ||
513 | async def set_user_perms(self, username, permissions): | 498 | async def set_user_perms(self, username, permissions): |
514 | statement = ( | ||
515 | update(Users) | ||
516 | .where(Users.username == username) | ||
517 | .values(permissions=" ".join(permissions)) | ||
518 | ) | ||
519 | self.logger.debug("%s", statement) | ||
520 | async with self.db.begin(): | 499 | async with self.db.begin(): |
521 | result = await self.db.execute(statement) | 500 | result = await self._execute( |
501 | update(Users) | ||
502 | .where(Users.username == username) | ||
503 | .values(permissions=" ".join(permissions)) | ||
504 | ) | ||
522 | return result.rowcount != 0 | 505 | return result.rowcount != 0 |
523 | 506 | ||
524 | async def get_all_users(self): | 507 | async def get_all_users(self): |
525 | statement = select( | ||
526 | Users.username, | ||
527 | Users.permissions, | ||
528 | ) | ||
529 | self.logger.debug("%s", statement) | ||
530 | async with self.db.begin(): | 508 | async with self.db.begin(): |
531 | result = await self.db.execute(statement) | 509 | result = await self._execute( |
510 | select( | ||
511 | Users.username, | ||
512 | Users.permissions, | ||
513 | ) | ||
514 | ) | ||
532 | return [map_user(row) for row in result] | 515 | return [map_user(row) for row in result] |
533 | 516 | ||
534 | async def new_user(self, username, permissions, token): | 517 | async def new_user(self, username, permissions, token): |
535 | statement = insert(Users).values( | ||
536 | username=username, | ||
537 | permissions=" ".join(permissions), | ||
538 | token=token, | ||
539 | ) | ||
540 | self.logger.debug("%s", statement) | ||
541 | try: | 518 | try: |
542 | async with self.db.begin(): | 519 | async with self.db.begin(): |
543 | await self.db.execute(statement) | 520 | await self._execute( |
521 | insert(Users).values( | ||
522 | username=username, | ||
523 | permissions=" ".join(permissions), | ||
524 | token=token, | ||
525 | ) | ||
526 | ) | ||
544 | return True | 527 | return True |
545 | except IntegrityError as e: | 528 | except IntegrityError as e: |
546 | self.logger.debug("Cannot create new user %s: %s", username, e) | 529 | self.logger.debug("Cannot create new user %s: %s", username, e) |
547 | return False | 530 | return False |
548 | 531 | ||
549 | async def delete_user(self, username): | 532 | async def delete_user(self, username): |
550 | statement = delete(Users).where(Users.username == username) | ||
551 | self.logger.debug("%s", statement) | ||
552 | async with self.db.begin(): | 533 | async with self.db.begin(): |
553 | result = await self.db.execute(statement) | 534 | result = await self._execute( |
535 | delete(Users).where(Users.username == username) | ||
536 | ) | ||
554 | return result.rowcount != 0 | 537 | return result.rowcount != 0 |
555 | 538 | ||
556 | async def get_usage(self): | 539 | async def get_usage(self): |
557 | usage = {} | 540 | usage = {} |
558 | async with self.db.begin() as session: | 541 | async with self.db.begin() as session: |
559 | for name, table in Base.metadata.tables.items(): | 542 | for name, table in Base.metadata.tables.items(): |
560 | statement = select(func.count()).select_from(table) | 543 | result = await self._execute( |
561 | self.logger.debug("%s", statement) | 544 | statement=select(func.count()).select_from(table) |
562 | result = await self.db.execute(statement) | 545 | ) |
563 | usage[name] = { | 546 | usage[name] = { |
564 | "rows": result.scalar(), | 547 | "rows": result.scalar(), |
565 | } | 548 | } |