diff options
author | Tobias Hagelborn <tobias.hagelborn@axis.com> | 2024-02-18 15:59:53 -0700 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2024-02-19 11:58:12 +0000 |
commit | 4e673ccb00d9d03150acc99b2c6d395d3c540b34 (patch) | |
tree | ab9c1c9e48852899a99f3fecb1337aacf58f59eb | |
parent | 61e184b3ed4d9691af36917036ceeab2bc8482f9 (diff) | |
download | poky-4e673ccb00d9d03150acc99b2c6d395d3c540b34.tar.gz |
bitbake: bitbake: hashserv: Postgres adaptations for ignoring duplicate inserts
Hash Equivalence server performs unconditional insert also of duplicate
hash entries. This causes excessive error log entries in Postgres.
Rather ignore the duplicate inserts.
The alternate behavior should be isolated to the postgres
engine type.
(Bitbake rev: e8d2d178d0fe96f9d6031c97328e8be17d752716)
Signed-off-by: Tobias Hagelborn <tobiasha@axis.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/hashserv/sqlalchemy.py | 53 |
1 files changed, 38 insertions, 15 deletions
diff --git a/bitbake/lib/hashserv/sqlalchemy.py b/bitbake/lib/hashserv/sqlalchemy.py index 0e28d738f5..fc3ae3d339 100644 --- a/bitbake/lib/hashserv/sqlalchemy.py +++ b/bitbake/lib/hashserv/sqlalchemy.py | |||
@@ -33,6 +33,7 @@ from sqlalchemy import ( | |||
33 | import sqlalchemy.engine | 33 | import sqlalchemy.engine |
34 | from sqlalchemy.orm import declarative_base | 34 | from sqlalchemy.orm import declarative_base |
35 | from sqlalchemy.exc import IntegrityError | 35 | from sqlalchemy.exc import IntegrityError |
36 | from sqlalchemy.dialects.postgresql import insert as postgres_insert | ||
36 | 37 | ||
37 | Base = declarative_base() | 38 | Base = declarative_base() |
38 | 39 | ||
@@ -283,9 +284,7 @@ class Database(object): | |||
283 | async def unihash_exists(self, unihash): | 284 | async def unihash_exists(self, unihash): |
284 | async with self.db.begin(): | 285 | async with self.db.begin(): |
285 | result = await self._execute( | 286 | result = await self._execute( |
286 | select(UnihashesV3) | 287 | select(UnihashesV3).where(UnihashesV3.unihash == unihash).limit(1) |
287 | .where(UnihashesV3.unihash == unihash) | ||
288 | .limit(1) | ||
289 | ) | 288 | ) |
290 | 289 | ||
291 | return result.first() is not None | 290 | return result.first() is not None |
@@ -435,18 +434,30 @@ class Database(object): | |||
435 | return result.rowcount | 434 | return result.rowcount |
436 | 435 | ||
437 | async def insert_unihash(self, method, taskhash, unihash): | 436 | async def insert_unihash(self, method, taskhash, unihash): |
438 | try: | 437 | # Postgres specific ignore on insert duplicate |
439 | async with self.db.begin(): | 438 | if self.engine.name == "postgresql": |
440 | await self._execute( | 439 | statement = ( |
441 | insert(UnihashesV3).values( | 440 | postgres_insert(UnihashesV3) |
442 | method=method, | 441 | .values( |
443 | taskhash=taskhash, | 442 | method=method, |
444 | unihash=unihash, | 443 | taskhash=taskhash, |
445 | gc_mark=self._get_config_subquery("gc-mark", ""), | 444 | unihash=unihash, |
446 | ) | 445 | gc_mark=self._get_config_subquery("gc-mark", ""), |
447 | ) | 446 | ) |
447 | .on_conflict_do_nothing(index_elements=("method", "taskhash")) | ||
448 | ) | ||
449 | else: | ||
450 | statement = insert(UnihashesV3).values( | ||
451 | method=method, | ||
452 | taskhash=taskhash, | ||
453 | unihash=unihash, | ||
454 | gc_mark=self._get_config_subquery("gc-mark", ""), | ||
455 | ) | ||
448 | 456 | ||
449 | return True | 457 | try: |
458 | async with self.db.begin(): | ||
459 | result = await self._execute(statement) | ||
460 | return result.rowcount != 0 | ||
450 | except IntegrityError: | 461 | except IntegrityError: |
451 | self.logger.debug( | 462 | self.logger.debug( |
452 | "%s, %s, %s already in unihash database", method, taskhash, unihash | 463 | "%s, %s, %s already in unihash database", method, taskhash, unihash |
@@ -461,10 +472,22 @@ class Database(object): | |||
461 | if "created" in data and not isinstance(data["created"], datetime): | 472 | if "created" in data and not isinstance(data["created"], datetime): |
462 | data["created"] = datetime.fromisoformat(data["created"]) | 473 | data["created"] = datetime.fromisoformat(data["created"]) |
463 | 474 | ||
475 | # Postgres specific ignore on insert duplicate | ||
476 | if self.engine.name == "postgresql": | ||
477 | statement = ( | ||
478 | postgres_insert(OuthashesV2) | ||
479 | .values(**data) | ||
480 | .on_conflict_do_nothing( | ||
481 | index_elements=("method", "taskhash", "outhash") | ||
482 | ) | ||
483 | ) | ||
484 | else: | ||
485 | statement = insert(OuthashesV2).values(**data) | ||
486 | |||
464 | try: | 487 | try: |
465 | async with self.db.begin(): | 488 | async with self.db.begin(): |
466 | await self._execute(insert(OuthashesV2).values(**data)) | 489 | result = await self._execute(statement) |
467 | return True | 490 | return result.rowcount != 0 |
468 | except IntegrityError: | 491 | except IntegrityError: |
469 | self.logger.debug( | 492 | self.logger.debug( |
470 | "%s, %s already in outhash database", data["method"], data["outhash"] | 493 | "%s, %s already in outhash database", data["method"], data["outhash"] |