diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-08-24 17:12:30 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-08-25 18:14:53 +0100 |
commit | 7002d67de02206fca0782c3373e915de20a71e9e (patch) | |
tree | ea6b56cbd121d617801220dc42ed12bb1399fcff /bitbake/lib | |
parent | 6bab132879c0abf6f2b1670174445766198d3cac (diff) | |
download | poky-7002d67de02206fca0782c3373e915de20a71e9e.tar.gz |
bitbake: server/process: Add bitbake-server and exec() a new server process
Trying to have a new python process forked off an original doesn't work
out well and ends up having race issues. To avoid this, exec() a new
bitbake server process. This starts with a fresh python interpreter
and resolves various atexit and other multiprocessing issues once
and for all.
(Bitbake rev: 9501dd6fdd7a7c25cbfa4464cf881fcf8c049ce2)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/server/process.py | 94 |
1 files changed, 55 insertions, 39 deletions
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index 915651084e..005ac34775 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py | |||
@@ -38,7 +38,7 @@ class ProcessServer(): | |||
38 | profile_filename = "profile.log" | 38 | profile_filename = "profile.log" |
39 | profile_processed_filename = "profile.log.processed" | 39 | profile_processed_filename = "profile.log.processed" |
40 | 40 | ||
41 | def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface): | 41 | def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface): |
42 | self.command_channel = False | 42 | self.command_channel = False |
43 | self.command_channel_reply = False | 43 | self.command_channel_reply = False |
44 | self.quit = False | 44 | self.quit = False |
@@ -54,6 +54,7 @@ class ProcessServer(): | |||
54 | self._idlefuns = {} | 54 | self._idlefuns = {} |
55 | 55 | ||
56 | self.bitbake_lock = lock | 56 | self.bitbake_lock = lock |
57 | self.bitbake_lock_name = lockname | ||
57 | self.sock = sock | 58 | self.sock = sock |
58 | self.sockname = sockname | 59 | self.sockname = sockname |
59 | 60 | ||
@@ -259,7 +260,7 @@ class ProcessServer(): | |||
259 | 260 | ||
260 | # Finally release the lockfile but warn about other processes holding it open | 261 | # Finally release the lockfile but warn about other processes holding it open |
261 | lock = self.bitbake_lock | 262 | lock = self.bitbake_lock |
262 | lockfile = lock.name | 263 | lockfile = self.bitbake_lock_name |
263 | lock.close() | 264 | lock.close() |
264 | lock = None | 265 | lock = None |
265 | 266 | ||
@@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object): | |||
393 | self.connection.recv.close() | 394 | self.connection.recv.close() |
394 | return | 395 | return |
395 | 396 | ||
397 | start_log_format = '--- Starting bitbake server pid %s at %s ---' | ||
398 | start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' | ||
399 | |||
396 | class BitBakeServer(object): | 400 | class BitBakeServer(object): |
397 | start_log_format = '--- Starting bitbake server pid %s at %s ---' | ||
398 | start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' | ||
399 | 401 | ||
400 | def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): | 402 | def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): |
401 | 403 | ||
@@ -408,6 +410,7 @@ class BitBakeServer(object): | |||
408 | 410 | ||
409 | # Place the log in the builddirectory alongside the lock file | 411 | # Place the log in the builddirectory alongside the lock file |
410 | logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") | 412 | logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") |
413 | self.logfile = logfile | ||
411 | 414 | ||
412 | startdatetime = datetime.datetime.now() | 415 | startdatetime = datetime.datetime.now() |
413 | bb.daemonize.createDaemon(self._startServer, logfile) | 416 | bb.daemonize.createDaemon(self._startServer, logfile) |
@@ -429,7 +432,7 @@ class BitBakeServer(object): | |||
429 | ready.close() | 432 | ready.close() |
430 | bb.error("Unable to start bitbake server (%s)" % str(r)) | 433 | bb.error("Unable to start bitbake server (%s)" % str(r)) |
431 | if os.path.exists(logfile): | 434 | if os.path.exists(logfile): |
432 | logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) | 435 | logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) |
433 | started = False | 436 | started = False |
434 | lines = [] | 437 | lines = [] |
435 | lastlines = [] | 438 | lastlines = [] |
@@ -441,7 +444,7 @@ class BitBakeServer(object): | |||
441 | lastlines.append(line) | 444 | lastlines.append(line) |
442 | res = logstart_re.match(line.rstrip()) | 445 | res = logstart_re.match(line.rstrip()) |
443 | if res: | 446 | if res: |
444 | ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) | 447 | ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format) |
445 | if ldatetime >= startdatetime: | 448 | if ldatetime >= startdatetime: |
446 | started = True | 449 | started = True |
447 | lines.append(line) | 450 | lines.append(line) |
@@ -462,42 +465,55 @@ class BitBakeServer(object): | |||
462 | ready.close() | 465 | ready.close() |
463 | 466 | ||
464 | def _startServer(self): | 467 | def _startServer(self): |
465 | print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format))) | 468 | os.close(self.readypipe) |
466 | sys.stdout.flush() | 469 | os.set_inheritable(self.bitbake_lock.fileno(), True) |
470 | os.set_inheritable(self.readypipein, True) | ||
471 | serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") | ||
472 | os.execl(serverscript, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) | ||
467 | 473 | ||
468 | try: | 474 | def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface): |
469 | # Create server control socket | ||
470 | if os.path.exists(self.sockname): | ||
471 | os.unlink(self.sockname) | ||
472 | 475 | ||
473 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | 476 | import bb.cookerdata |
474 | # AF_UNIX has path length issues so chdir here to workaround | 477 | import bb.cooker |
475 | cwd = os.getcwd() | 478 | |
476 | try: | 479 | print(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format))) |
477 | os.chdir(os.path.dirname(self.sockname)) | 480 | sys.stdout.flush() |
478 | sock.bind(os.path.basename(self.sockname)) | 481 | |
479 | finally: | 482 | try: |
480 | os.chdir(cwd) | 483 | bitbake_lock = os.fdopen(lockfd, "w") |
481 | sock.listen(1) | 484 | |
482 | 485 | # Create server control socket | |
483 | server = ProcessServer(self.bitbake_lock, sock, self.sockname, self.server_timeout, self.xmlrpcinterface) | 486 | if os.path.exists(sockname): |
484 | os.close(self.readypipe) | 487 | os.unlink(sockname) |
485 | writer = ConnectionWriter(self.readypipein) | 488 | |
486 | try: | 489 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
487 | self.cooker = bb.cooker.BBCooker(self.featureset, server.register_idle_function) | 490 | # AF_UNIX has path length issues so chdir here to workaround |
488 | except bb.BBHandledException: | 491 | cwd = os.getcwd() |
489 | return None | 492 | try: |
490 | writer.send("r") | 493 | os.chdir(os.path.dirname(sockname)) |
491 | writer.close() | 494 | sock.bind(os.path.basename(sockname)) |
492 | server.cooker = self.cooker | ||
493 | print("Started bitbake server pid %d" % os.getpid()) | ||
494 | sys.stdout.flush() | ||
495 | |||
496 | server.run() | ||
497 | finally: | 495 | finally: |
498 | # Flush any ,essages/errors to the logfile before exit | 496 | os.chdir(cwd) |
499 | sys.stdout.flush() | 497 | sock.listen(1) |
500 | sys.stderr.flush() | 498 | |
499 | server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface) | ||
500 | writer = ConnectionWriter(readypipeinfd) | ||
501 | try: | ||
502 | featureset = [] | ||
503 | cooker = bb.cooker.BBCooker(featureset, server.register_idle_function) | ||
504 | except bb.BBHandledException: | ||
505 | return None | ||
506 | writer.send("r") | ||
507 | writer.close() | ||
508 | server.cooker = cooker | ||
509 | print("Started bitbake server pid %d" % os.getpid()) | ||
510 | sys.stdout.flush() | ||
511 | |||
512 | server.run() | ||
513 | finally: | ||
514 | # Flush any ,essages/errors to the logfile before exit | ||
515 | sys.stdout.flush() | ||
516 | sys.stderr.flush() | ||
501 | 517 | ||
502 | def connectProcessServer(sockname, featureset): | 518 | def connectProcessServer(sockname, featureset): |
503 | # Connect to socket | 519 | # Connect to socket |