summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-08-24 17:12:30 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-08-25 18:14:53 +0100
commit7002d67de02206fca0782c3373e915de20a71e9e (patch)
treeea6b56cbd121d617801220dc42ed12bb1399fcff /bitbake
parent6bab132879c0abf6f2b1670174445766198d3cac (diff)
downloadpoky-7002d67de02206fca0782c3373e915de20a71e9e.tar.gz
bitbake: server/process: Add bitbake-server and exec() a new server process
Trying to have a new python process forked off an original doesn't work out well and ends up having race issues. To avoid this, exec() a new bitbake server process. This starts with a fresh python interpreter and resolves various atexit and other multiprocessing issues once and for all. (Bitbake rev: 9501dd6fdd7a7c25cbfa4464cf881fcf8c049ce2) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rwxr-xr-xbitbake/bin/bitbake-server54
-rw-r--r--bitbake/lib/bb/server/process.py94
2 files changed, 109 insertions, 39 deletions
diff --git a/bitbake/bin/bitbake-server b/bitbake/bin/bitbake-server
new file mode 100755
index 0000000000..ffbc7894ef
--- /dev/null
+++ b/bitbake/bin/bitbake-server
@@ -0,0 +1,54 @@
1#!/usr/bin/env python3
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# Copyright (C) 2020 Richard Purdie
6#
7
8import os
9import sys
10import warnings
11import logging
12sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
13
14if sys.getfilesystemencoding() != "utf-8":
15 sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
16
17# Users shouldn't be running this code directly
18if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
19 print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.")
20 sys.exit(1)
21
22import bb.server.process
23
24lockfd = int(sys.argv[2])
25readypipeinfd = int(sys.argv[3])
26logfile = sys.argv[4]
27lockname = sys.argv[5]
28sockname = sys.argv[6]
29timeout = sys.argv[7]
30xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
31if xmlrpcinterface[0] == "None":
32 xmlrpcinterface = (None, xmlrpcinterface[1])
33if timeout == "None":
34 timeout = None
35
36# Replace standard fds with our own
37with open('/dev/null', 'r') as si:
38 os.dup2(si.fileno(), sys.stdin.fileno())
39
40so = open(logfile, 'a+')
41os.dup2(so.fileno(), sys.stdout.fileno())
42os.dup2(so.fileno(), sys.stderr.fileno())
43
44# Have stdout and stderr be the same so log output matches chronologically
45# and there aren't two seperate buffers
46sys.stderr = sys.stdout
47
48logger = logging.getLogger("BitBake")
49# Ensure logging messages get sent to the UI as events
50handler = bb.event.LogHandler()
51logger.addHandler(handler)
52
53bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface)
54
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 915651084e..005ac34775 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -38,7 +38,7 @@ class ProcessServer():
38 profile_filename = "profile.log" 38 profile_filename = "profile.log"
39 profile_processed_filename = "profile.log.processed" 39 profile_processed_filename = "profile.log.processed"
40 40
41 def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface): 41 def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
42 self.command_channel = False 42 self.command_channel = False
43 self.command_channel_reply = False 43 self.command_channel_reply = False
44 self.quit = False 44 self.quit = False
@@ -54,6 +54,7 @@ class ProcessServer():
54 self._idlefuns = {} 54 self._idlefuns = {}
55 55
56 self.bitbake_lock = lock 56 self.bitbake_lock = lock
57 self.bitbake_lock_name = lockname
57 self.sock = sock 58 self.sock = sock
58 self.sockname = sockname 59 self.sockname = sockname
59 60
@@ -259,7 +260,7 @@ class ProcessServer():
259 260
260 # Finally release the lockfile but warn about other processes holding it open 261 # Finally release the lockfile but warn about other processes holding it open
261 lock = self.bitbake_lock 262 lock = self.bitbake_lock
262 lockfile = lock.name 263 lockfile = self.bitbake_lock_name
263 lock.close() 264 lock.close()
264 lock = None 265 lock = None
265 266
@@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
393 self.connection.recv.close() 394 self.connection.recv.close()
394 return 395 return
395 396
397start_log_format = '--- Starting bitbake server pid %s at %s ---'
398start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
399
396class BitBakeServer(object): 400class BitBakeServer(object):
397 start_log_format = '--- Starting bitbake server pid %s at %s ---'
398 start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
399 401
400 def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): 402 def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
401 403
@@ -408,6 +410,7 @@ class BitBakeServer(object):
408 410
409 # Place the log in the builddirectory alongside the lock file 411 # Place the log in the builddirectory alongside the lock file
410 logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") 412 logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
413 self.logfile = logfile
411 414
412 startdatetime = datetime.datetime.now() 415 startdatetime = datetime.datetime.now()
413 bb.daemonize.createDaemon(self._startServer, logfile) 416 bb.daemonize.createDaemon(self._startServer, logfile)
@@ -429,7 +432,7 @@ class BitBakeServer(object):
429 ready.close() 432 ready.close()
430 bb.error("Unable to start bitbake server (%s)" % str(r)) 433 bb.error("Unable to start bitbake server (%s)" % str(r))
431 if os.path.exists(logfile): 434 if os.path.exists(logfile):
432 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) 435 logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
433 started = False 436 started = False
434 lines = [] 437 lines = []
435 lastlines = [] 438 lastlines = []
@@ -441,7 +444,7 @@ class BitBakeServer(object):
441 lastlines.append(line) 444 lastlines.append(line)
442 res = logstart_re.match(line.rstrip()) 445 res = logstart_re.match(line.rstrip())
443 if res: 446 if res:
444 ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) 447 ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
445 if ldatetime >= startdatetime: 448 if ldatetime >= startdatetime:
446 started = True 449 started = True
447 lines.append(line) 450 lines.append(line)
@@ -462,42 +465,55 @@ class BitBakeServer(object):
462 ready.close() 465 ready.close()
463 466
464 def _startServer(self): 467 def _startServer(self):
465 print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format))) 468 os.close(self.readypipe)
466 sys.stdout.flush() 469 os.set_inheritable(self.bitbake_lock.fileno(), True)
470 os.set_inheritable(self.readypipein, True)
471 serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
472 os.execl(serverscript, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
467 473
468 try: 474def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
469 # Create server control socket
470 if os.path.exists(self.sockname):
471 os.unlink(self.sockname)
472 475
473 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 476 import bb.cookerdata
474 # AF_UNIX has path length issues so chdir here to workaround 477 import bb.cooker
475 cwd = os.getcwd() 478
476 try: 479 print(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
477 os.chdir(os.path.dirname(self.sockname)) 480 sys.stdout.flush()
478 sock.bind(os.path.basename(self.sockname)) 481
479 finally: 482 try:
480 os.chdir(cwd) 483 bitbake_lock = os.fdopen(lockfd, "w")
481 sock.listen(1) 484
482 485 # Create server control socket
483 server = ProcessServer(self.bitbake_lock, sock, self.sockname, self.server_timeout, self.xmlrpcinterface) 486 if os.path.exists(sockname):
484 os.close(self.readypipe) 487 os.unlink(sockname)
485 writer = ConnectionWriter(self.readypipein) 488
486 try: 489 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
487 self.cooker = bb.cooker.BBCooker(self.featureset, server.register_idle_function) 490 # AF_UNIX has path length issues so chdir here to workaround
488 except bb.BBHandledException: 491 cwd = os.getcwd()
489 return None 492 try:
490 writer.send("r") 493 os.chdir(os.path.dirname(sockname))
491 writer.close() 494 sock.bind(os.path.basename(sockname))
492 server.cooker = self.cooker
493 print("Started bitbake server pid %d" % os.getpid())
494 sys.stdout.flush()
495
496 server.run()
497 finally: 495 finally:
498 # Flush any ,essages/errors to the logfile before exit 496 os.chdir(cwd)
499 sys.stdout.flush() 497 sock.listen(1)
500 sys.stderr.flush() 498
499 server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
500 writer = ConnectionWriter(readypipeinfd)
501 try:
502 featureset = []
503 cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
504 except bb.BBHandledException:
505 return None
506 writer.send("r")
507 writer.close()
508 server.cooker = cooker
509 print("Started bitbake server pid %d" % os.getpid())
510 sys.stdout.flush()
511
512 server.run()
513 finally:
514 # Flush any ,essages/errors to the logfile before exit
515 sys.stdout.flush()
516 sys.stderr.flush()
501 517
502def connectProcessServer(sockname, featureset): 518def connectProcessServer(sockname, featureset):
503 # Connect to socket 519 # Connect to socket