6 files changed, 40 insertions, 36 deletions

diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index ed266f0ac2..a3ea5d9618 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -121,11 +121,10 @@ def worker_child_fire(event, d):
 
     data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
-        worker_pipe_lock.acquire()
-        while(len(data)):
-            written = worker_pipe.write(data)
-            data = data[written:]
-        worker_pipe_lock.release()
+        with bb.utils.lock_timeout(worker_pipe_lock):
+            while(len(data)):
+                written = worker_pipe.write(data)
+                data = data[written:]
     except IOError:
         sigterm_handler(None, None)
         raise
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index a5a635858c..738849d189 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -251,14 +251,14 @@ class BBCooker:
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
     def process_inotify_updates(self):
-        with self.inotify_threadlock:
+        with bb.utils.lock_timeout(self.inotify_threadlock):
             for n in [self.confignotifier, self.notifier]:
                 if n and n.check_events(timeout=0):
                     # read notified events and enqueue them
                     n.read_events()
 
     def process_inotify_updates_apply(self):
-        with self.inotify_threadlock:
+        with bb.utils.lock_timeout(self.inotify_threadlock):
             for n in [self.confignotifier, self.notifier]:
                 if n and n.check_events(timeout=0):
                     n.read_events()
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 7826541a64..8b05f93e2f 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -184,7 +184,7 @@ def fire_ui_handlers(event, d):
         ui_queue.append(event)
         return
 
-    with _thread_lock:
+    with bb.utils.lock_timeout(_thread_lock):
         errors = []
         for h in _ui_handlers:
             #print "Sending event %s" % event
@@ -315,7 +315,7 @@ def set_eventfilter(func):
     _eventfilter = func
 
 def register_UIHhandler(handler, mainui=False):
-    with _thread_lock:
+    with bb.utils.lock_timeout(_thread_lock):
         bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
         _ui_handlers[_ui_handler_seq] = handler
         level, debug_domains = bb.msg.constructLogOptions()
@@ -329,7 +329,7 @@ def unregister_UIHhandler(handlerNum, mainui=False):
     if mainui:
         global _uiready
         _uiready = False
-    with _thread_lock:
+    with bb.utils.lock_timeout(_thread_lock):
         if handlerNum in _ui_handlers:
             del _ui_handlers[handlerNum]
     return
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index ac7749d36c..b5f6faf6fb 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -113,7 +113,7 @@ class ProcessServer():
     def register_idle_function(self, function, data):
         """Register a function to be called while the server is idle"""
         assert hasattr(function, '__call__')
-        with self._idlefuncsLock:
+        with bb.utils.lock_timeout(self._idlefuncsLock):
             self._idlefuns[function] = data
         serverlog("Registering idle function %s" % str(function))
 
@@ -379,7 +379,7 @@ class ProcessServer():
 
     def idle_thread(self):
         def remove_idle_func(function):
-            with self._idlefuncsLock:
+            with bb.utils.lock_timeout(self._idlefuncsLock):
                 del self._idlefuns[function]
                 self.idle_cond.notify_all()
 
@@ -387,7 +387,7 @@ class ProcessServer():
             nextsleep = 0.1
             fds = []
 
-            with self._idlefuncsLock:
+            with bb.utils.lock_timeout(self._idlefuncsLock):
                 items = list(self._idlefuns.items())
 
             for function, data in items:
@@ -743,7 +743,7 @@ class BBUIEventQueue:
         self.t.start()
 
     def getEvent(self):
-        with self.eventQueueLock:
+        with bb.utils.lock_timeout(self.eventQueueLock):
             if len(self.eventQueue) == 0:
                 return None
 
@@ -758,7 +758,7 @@ class BBUIEventQueue:
         return self.getEvent()
 
     def queue_event(self, event):
-        with self.eventQueueLock:
+        with bb.utils.lock_timeout(self.eventQueueLock):
             self.eventQueue.append(event)
             self.eventQueueNotify.set()
 
@@ -794,7 +794,7 @@ class ConnectionReader(object):
         return self.reader.poll(timeout)
 
     def get(self):
-        with self.rlock:
+        with bb.utils.lock_timeout(self.rlock):
             res = self.reader.recv_bytes()
         return multiprocessing.reduction.ForkingPickler.loads(res)
 
@@ -815,7 +815,7 @@ class ConnectionWriter(object):
 
     def _send(self, obj):
         gc.disable()
-        with self.wlock:
+        with bb.utils.lock_timeout(self.wlock):
             self.writer.send_bytes(obj)
         gc.enable()
 
@@ -828,7 +828,7 @@ class ConnectionWriter(object):
         # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
         process = multiprocessing.current_process()
         if process and hasattr(process, "queue_signals"):
-            with process.signal_threadlock:
+            with bb.utils.lock_timeout(process.signal_threadlock):
                 process.queue_signals = True
                 self._send(obj)
                 process.queue_signals = False
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index d595f172ac..adbe698939 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -70,30 +70,22 @@ class BBUIEventQueue:
         self.t.start()
 
     def getEvent(self):
-
-        self.eventQueueLock.acquire()
-
-        if not self.eventQueue:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
-
-        if not self.eventQueue:
-            self.eventQueueNotify.clear()
-
-        self.eventQueueLock.release()
-        return item
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            if not self.eventQueue:
+                return None
+            item = self.eventQueue.pop(0)
+            if not self.eventQueue:
+                self.eventQueueNotify.clear()
+            return item
 
     def waitEvent(self, delay):
         self.eventQueueNotify.wait(delay)
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 0df522b372..8c79159573 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -1841,3 +1841,16 @@ def mkstemp(suffix=None, prefix=None, dir=None, text=False):
     else:
         prefix = tempfile.gettempprefix() + entropy
     return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
+
+# If we don't have a timeout of some kind and a process/thread exits badly (for example
+# OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
+# we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.
+@contextmanager
+def lock_timeout(lock):
+    held = lock.acquire(timeout=5*60)
+    try:
+        if not held:
+            os._exit(1)
+        yield held
+    finally:
+        lock.release()