bitbake: server/process: Improve idle thread exception handling

If the inotifier code has an exception, bitbake currently hangs. Catch any
exception and exit if seen. Also check the idle thread is alive and exit
if it disappears. This should stop bitbake hanging if such a situation arises
in future such as this example:

3323260 21:48:31.554468 Running command ['getVariable', 'BBINCLUDELOGS']
Exception in thread Thread-1 (idle_thread):
Traceback (most recent call last):
  File "/usr/lib64/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/usr/lib64/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/server/process.py", line 408, in idle_thread
    self.cooker.process_inotify_updates()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/cooker.py", line 256, in process_inotify_updates
    n.read_events()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/pyinotify.py", line 1207, in read_events
    if fcntl.ioctl(self._fd, termios.FIONREAD, buf_, 1) == -1:
OSError: [Errno 9] Bad file descriptor
3323260 21:48:32.206995 Command Completed (socket: True)

(Bitbake rev: 358b5b02d5de1ab0f98104c4ec4953e46999b9a5)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 9 insertions, 1 deletions

diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 916ee0a0e5..db417c8428 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -405,7 +405,11 @@ class ProcessServer():
             nextsleep = 0.1
             fds = []
 
-            self.cooker.process_inotify_updates()
+            try:
+                self.cooker.process_inotify_updates()
+            except Exception as exc:
+                serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
+                self.quit = True
 
             with bb.utils.lock_timeout(self._idlefuncsLock):
                 items = list(self._idlefuns.items())
@@ -473,6 +477,10 @@ class ProcessServer():
         if not self.idle:
             self.idle = threading.Thread(target=self.idle_thread)
             self.idle.start()
+        elif self.idle and not self.idle.is_alive():
+            serverlog("Idle thread terminated, main thread exiting too")
+            bb.error("Idle thread terminated, main thread exiting too")
+            self.quit = True
 
         if nextsleep is not None:
             if self.xmlrpc: