summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2021-05-17 13:57:02 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-05-18 23:53:15 +0100
commit2f12a20935acec5218e38efc1a97c686c06f31ee (patch)
treefbf458e396c94eccd9968d57e9678972803f2d99 /bitbake
parente109bdae37b6514384de5ad9f93fb3229a0e1a45 (diff)
downloadpoky-2f12a20935acec5218e38efc1a97c686c06f31ee.tar.gz
bitbake: server/process: Handle error in heartbeat funciton in OOM case
We've seen cases where an OOM error causes bitbake server to hang: 9171 02:21:09.127810 Command Completed Traceback (most recent call last): File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/bin/bitbake-server", line 51, in <module> bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface) File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/server/process.py", line 550, in execServer server.run() File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/server/process.py", line 108, in run ret = self.main() File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/server/process.py", line 242, in main ready = self.idle_commands(.1, fds) File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/server/process.py", line 370, in idle_commands bb.event.fire(heartbeat, self.cooker.data) File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/event.py", line 216, in fire fire_class_handlers(event, d) File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/event.py", line 123, in fire_class_handlers execute_handler(name, handler, event, d) File "/home/pokybuild/yocto-worker/qemux86/build/bitbake/lib/bb/event.py", line 93, in execute_handler ret = handler(event) File "/home/pokybuild/yocto-worker/qemux86/build/meta/classes/buildstats.bbclass", line 182, in defaultrun_buildstats write_host_data(os.path.join(bsdir, "host_stats"), e, d, "interval") File "/home/pokybuild/yocto-worker/qemux86/build/meta/classes/buildstats.bbclass", line 160, in write_host_data output = subprocess.check_output(c.split(), stderr=subprocess.STDOUT, timeout=limit).decode('utf-8') File "/usr/lib/python3.6/subprocess.py", line 356, in check_output **kwargs).stdout File "/usr/lib/python3.6/subprocess.py", line 423, in run with Popen(*popenargs, **kwargs) as process: File "/usr/lib/python3.6/subprocess.py", line 729, in __init__ restore_signals, start_new_session) File "/usr/lib/python3.6/subprocess.py", line 1295, in _execute_child restore_signals, start_new_session, preexec_fn) OSError: [Errno 12] Cannot allocate memory We need to wrap the calls in the same high level wrapper as idle function calls and trigger an exit upon an unhandled exception. (Bitbake rev: 74042b5b89d5a170013fc1a327ce3a6530fbf7d5) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/server/process.py7
1 files changed, 6 insertions, 1 deletions
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 3e99bcef8f..155e8d131f 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -367,7 +367,12 @@ class ProcessServer():
367 self.next_heartbeat = now + self.heartbeat_seconds 367 self.next_heartbeat = now + self.heartbeat_seconds
368 if hasattr(self.cooker, "data"): 368 if hasattr(self.cooker, "data"):
369 heartbeat = bb.event.HeartbeatEvent(now) 369 heartbeat = bb.event.HeartbeatEvent(now)
370 bb.event.fire(heartbeat, self.cooker.data) 370 try:
371 bb.event.fire(heartbeat, self.cooker.data)
372 except Exception as exc:
373 if not isinstance(exc, bb.BBHandledException):
374 logger.exception('Running heartbeat function')
375 self.quit = True
371 if nextsleep and now + nextsleep > self.next_heartbeat: 376 if nextsleep and now + nextsleep > self.next_heartbeat:
372 # Shorten timeout so that we we wake up in time for 377 # Shorten timeout so that we we wake up in time for
373 # the heartbeat. 378 # the heartbeat.