diff options
author | Mark Asselstine <mark.asselstine@windriver.com> | 2023-12-28 16:01:17 -0500 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2024-01-10 14:02:38 +0000 |
commit | bc22d82c2f0c16ee8ad2edeb3bcdfb31db0afa94 (patch) | |
tree | 5b2ec6128e29913c02e4434e1756afc5b4b92b07 | |
parent | c665a2c93337fc372207af604c21bdea5babb4e5 (diff) | |
download | poky-bc22d82c2f0c16ee8ad2edeb3bcdfb31db0afa94.tar.gz |
bitbake: server/process: catch and expand multiprocessing connection exceptions
Doing builds on systems with limited resources, or with high demand
package builds such as chromium it isn't uncommon for the OOM Killer
to be triggered and for bitbake-server to be selected as the process
to be killed. When the bitbake-server does terminate unexpectedly due
to the OOM Killer or otherwise, this currently results in a generic
python traceback with little indication as to what has failed.
Here we trap and raise the exceptions while extending the exception
text in runCommand() to make it clear that this is most likely caused
by the bitbake-server unexpectedly terminating.
Callers of runCommand() should be updated to properly handle the
BrokenPipeError and EOFError exceptions to avoid printing a python
traceback, but even if they don't, the added text in the exceptions
should provide some hints as to what might have caused the failure.
(Bitbake rev: 5ff62b802f79acc86bbd6a99484f08501ff5dc2d)
Signed-off-by: Mark Asselstine <mark.asselstine@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/bb/server/process.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index d495ac6245..6d77ce4786 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py | |||
@@ -500,12 +500,18 @@ class ServerCommunicator(): | |||
500 | self.recv = recv | 500 | self.recv = recv |
501 | 501 | ||
502 | def runCommand(self, command): | 502 | def runCommand(self, command): |
503 | self.connection.send(command) | 503 | try: |
504 | self.connection.send(command) | ||
505 | except BrokenPipeError as e: | ||
506 | raise BrokenPipeError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e | ||
504 | if not self.recv.poll(30): | 507 | if not self.recv.poll(30): |
505 | logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime())) | 508 | logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime())) |
506 | if not self.recv.poll(30): | 509 | if not self.recv.poll(30): |
507 | raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime()) | 510 | raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime()) |
508 | ret, exc = self.recv.get() | 511 | try: |
512 | ret, exc = self.recv.get() | ||
513 | except EOFError as e: | ||
514 | raise EOFError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e | ||
509 | # Should probably turn all exceptions in exc back into exceptions? | 515 | # Should probably turn all exceptions in exc back into exceptions? |
510 | # For now, at least handle BBHandledException | 516 | # For now, at least handle BBHandledException |
511 | if exc and ("BBHandledException" in exc or "SystemExit" in exc): | 517 | if exc and ("BBHandledException" in exc or "SystemExit" in exc): |