diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2015-07-31 10:16:33 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2015-09-01 21:37:20 +0100 |
commit | fa87963b357c705597c61ffacb0f9c2ac12c1bd3 (patch) | |
tree | 1abee254eda11e38ec67fc2b44836ba27a8e00d7 | |
parent | d7cad22f482f8e5a8230572a23a0ca9800bfe7f3 (diff) | |
download | poky-fa87963b357c705597c61ffacb0f9c2ac12c1bd3.tar.gz |
bitbake: bitbake: cooker: properly fix bitbake.lock handling
If the PR server or indeed any other child process takes some time to
exit (which it sometimes does when saving its database), it can end up
holding bitbake.lock after the UI exits, which led to errors if you ran
bitbake commands successively - we saw this when running the PR server
oe-selftest tests in OE-Core. The recent attempt to fix this wasn't
quite right and ended up breaking memory resident bitbake. This time we
close the lock file when cooker shuts down (inside the UI process)
instead of unlocking it, and this is done in the cooker code rather than
the actual UI code so it doesn't matter which UI is in use. Additionally
we report that we're waiting for the lock to be released, using lsof or
fuser if available to list the processes with the lock open.
The 'magic' in the locking is due to all spawned subprocesses of bitbake
holding an open file descriptor to the bitbake.lock. It is automatically
unlocked when all those fds close the file (as all the processes terminate).
We close the UI copy of the lock explicitly, then close the server process
copy, any remaining open copy is therefore some proess exiting.
(The reproducer for the problem is to set PRSERV_HOST = "localhost:0"
and add a call to time.sleep(20) after self.server_close() in
lib/prserv/serv.py, then run "bitbake -p; bitbake -p" ).
Cleanup work done by Paul Eggleton <paul.eggleton@linux.intel.com>.
This reverts bitbake commit 69ecd15aece54753154950c55d7af42f85ad8606 and
e97a9f1528d77503b5c93e48e3de9933fbb9f3cd.
(Bitbake rev: a29780bd43f74b7326fe788dbd65177b86806fcf)
(Bitbake rev: ed30f4ee1cef8db9ea422c5e54b2375c4f3b1d6f)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Conflicts:
bitbake/lib/bb/tinfoil.py
-rw-r--r-- | bitbake/lib/bb/cooker.py | 30 | ||||
-rwxr-xr-x | bitbake/lib/bb/main.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/tinfoil.py | 5 | ||||
-rw-r--r-- | bitbake/lib/bb/utils.py | 29 |
4 files changed, 61 insertions, 4 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index f32dae0f58..d9bffc3ad2 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -38,6 +38,8 @@ import bb, bb.exceptions, bb.command | |||
38 | from bb import utils, data, parse, event, cache, providers, taskdata, runqueue | 38 | from bb import utils, data, parse, event, cache, providers, taskdata, runqueue |
39 | import Queue | 39 | import Queue |
40 | import signal | 40 | import signal |
41 | import subprocess | ||
42 | import errno | ||
41 | import prserv.serv | 43 | import prserv.serv |
42 | import pyinotify | 44 | import pyinotify |
43 | 45 | ||
@@ -1525,6 +1527,34 @@ class BBCooker: | |||
1525 | def post_serve(self): | 1527 | def post_serve(self): |
1526 | prserv.serv.auto_shutdown(self.data) | 1528 | prserv.serv.auto_shutdown(self.data) |
1527 | bb.event.fire(CookerExit(), self.expanded_data) | 1529 | bb.event.fire(CookerExit(), self.expanded_data) |
1530 | lockfile = self.lock.name | ||
1531 | self.lock.close() | ||
1532 | self.lock = None | ||
1533 | |||
1534 | while not self.lock: | ||
1535 | with bb.utils.timeout(3): | ||
1536 | self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) | ||
1537 | if not self.lock: | ||
1538 | # Some systems may not have lsof available | ||
1539 | procs = None | ||
1540 | try: | ||
1541 | procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) | ||
1542 | except OSError as e: | ||
1543 | if e.errno != errno.ENOENT: | ||
1544 | raise | ||
1545 | if procs is None: | ||
1546 | # Fall back to fuser if lsof is unavailable | ||
1547 | try: | ||
1548 | procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) | ||
1549 | except OSError as e: | ||
1550 | if e.errno != errno.ENOENT: | ||
1551 | raise | ||
1552 | |||
1553 | msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" | ||
1554 | if procs: | ||
1555 | msg += ":\n%s" % str(procs) | ||
1556 | print(msg) | ||
1557 | |||
1528 | 1558 | ||
1529 | def shutdown(self, force = False): | 1559 | def shutdown(self, force = False): |
1530 | if force: | 1560 | if force: |
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py index 1f19cc5dcf..fea754f534 100755 --- a/bitbake/lib/bb/main.py +++ b/bitbake/lib/bb/main.py | |||
@@ -258,6 +258,7 @@ def start_server(servermodule, configParams, configuration, features): | |||
258 | logger.handle(event) | 258 | logger.handle(event) |
259 | raise exc_info[1], None, exc_info[2] | 259 | raise exc_info[1], None, exc_info[2] |
260 | server.detach() | 260 | server.detach() |
261 | cooker.lock.close() | ||
261 | return server | 262 | return server |
262 | 263 | ||
263 | 264 | ||
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index 8fc9be3039..1ea46d8eec 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
@@ -84,6 +84,11 @@ class Tinfoil: | |||
84 | else: | 84 | else: |
85 | self.parseRecipes() | 85 | self.parseRecipes() |
86 | 86 | ||
87 | def shutdown(self): | ||
88 | self.cooker.shutdown(force=True) | ||
89 | self.cooker.post_serve() | ||
90 | self.cooker.unlockBitbake() | ||
91 | |||
87 | class TinfoilConfigParameters(ConfigParameters): | 92 | class TinfoilConfigParameters(ConfigParameters): |
88 | 93 | ||
89 | def __init__(self, **options): | 94 | def __init__(self, **options): |
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 0db7e56651..d380a3708c 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py | |||
@@ -31,6 +31,7 @@ import subprocess | |||
31 | import glob | 31 | import glob |
32 | import traceback | 32 | import traceback |
33 | import errno | 33 | import errno |
34 | import signal | ||
34 | from commands import getstatusoutput | 35 | from commands import getstatusoutput |
35 | from contextlib import contextmanager | 36 | from contextlib import contextmanager |
36 | 37 | ||
@@ -412,10 +413,30 @@ def fileslocked(files): | |||
412 | for lock in locks: | 413 | for lock in locks: |
413 | bb.utils.unlockfile(lock) | 414 | bb.utils.unlockfile(lock) |
414 | 415 | ||
415 | def lockfile(name, shared=False, retry=True): | 416 | @contextmanager |
417 | def timeout(seconds): | ||
418 | def timeout_handler(signum, frame): | ||
419 | pass | ||
420 | |||
421 | original_handler = signal.signal(signal.SIGALRM, timeout_handler) | ||
422 | |||
423 | try: | ||
424 | signal.alarm(seconds) | ||
425 | yield | ||
426 | finally: | ||
427 | signal.alarm(0) | ||
428 | signal.signal(signal.SIGALRM, original_handler) | ||
429 | |||
430 | def lockfile(name, shared=False, retry=True, block=False): | ||
416 | """ | 431 | """ |
417 | Use the file fn as a lock file, return when the lock has been acquired. | 432 | Use the specified file as a lock file, return when the lock has |
418 | Returns a variable to pass to unlockfile(). | 433 | been acquired. Returns a variable to pass to unlockfile(). |
434 | Parameters: | ||
435 | retry: True to re-try locking if it fails, False otherwise | ||
436 | block: True to block until the lock succeeds, False otherwise | ||
437 | The retry and block parameters are kind of equivalent unless you | ||
438 | consider the possibility of sending a signal to the process to break | ||
439 | out - at which point you want block=True rather than retry=True. | ||
419 | """ | 440 | """ |
420 | dirname = os.path.dirname(name) | 441 | dirname = os.path.dirname(name) |
421 | mkdirhier(dirname) | 442 | mkdirhier(dirname) |
@@ -428,7 +449,7 @@ def lockfile(name, shared=False, retry=True): | |||
428 | op = fcntl.LOCK_EX | 449 | op = fcntl.LOCK_EX |
429 | if shared: | 450 | if shared: |
430 | op = fcntl.LOCK_SH | 451 | op = fcntl.LOCK_SH |
431 | if not retry: | 452 | if not retry and not block: |
432 | op = op | fcntl.LOCK_NB | 453 | op = op | fcntl.LOCK_NB |
433 | 454 | ||
434 | while True: | 455 | while True: |