summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/main.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/main.py')
-rwxr-xr-xbitbake/lib/bb/main.py230
1 files changed, 77 insertions, 153 deletions
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py
index 29e391162e..1edf56f41b 100755
--- a/bitbake/lib/bb/main.py
+++ b/bitbake/lib/bb/main.py
@@ -28,6 +28,8 @@ import logging
28import optparse 28import optparse
29import warnings 29import warnings
30import fcntl 30import fcntl
31import time
32import traceback
31 33
32import bb 34import bb
33from bb import event 35from bb import event
@@ -37,6 +39,9 @@ from bb import ui
37from bb import server 39from bb import server
38from bb import cookerdata 40from bb import cookerdata
39 41
42import bb.server.process
43import bb.server.xmlrpcclient
44
40logger = logging.getLogger("BitBake") 45logger = logging.getLogger("BitBake")
41 46
42class BBMainException(Exception): 47class BBMainException(Exception):
@@ -58,9 +63,6 @@ class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
58 if option.dest == 'ui': 63 if option.dest == 'ui':
59 valid_uis = list_extension_modules(bb.ui, 'main') 64 valid_uis = list_extension_modules(bb.ui, 'main')
60 option.help = option.help.replace('@CHOICES@', present_options(valid_uis)) 65 option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
61 elif option.dest == 'servertype':
62 valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
63 option.help = option.help.replace('@CHOICES@', present_options(valid_server_types))
64 66
65 return optparse.IndentedHelpFormatter.format_option(self, option) 67 return optparse.IndentedHelpFormatter.format_option(self, option)
66 68
@@ -238,11 +240,6 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
238 default=os.environ.get('BITBAKE_UI', 'knotty'), 240 default=os.environ.get('BITBAKE_UI', 'knotty'),
239 help="The user interface to use (@CHOICES@ - default %default).") 241 help="The user interface to use (@CHOICES@ - default %default).")
240 242
241 # @CHOICES@ is substituted out by BitbakeHelpFormatter above
242 parser.add_option("-t", "--servertype", action="store", dest="servertype",
243 default=["process", "xmlrpc"]["BBSERVER" in os.environ],
244 help="Choose which server type to use (@CHOICES@ - default %default).")
245
246 parser.add_option("", "--token", action="store", dest="xmlrpctoken", 243 parser.add_option("", "--token", action="store", dest="xmlrpctoken",
247 default=os.environ.get("BBTOKEN"), 244 default=os.environ.get("BBTOKEN"),
248 help="Specify the connection token to be used when connecting " 245 help="Specify the connection token to be used when connecting "
@@ -258,14 +255,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
258 help="Run bitbake without a UI, only starting a server " 255 help="Run bitbake without a UI, only starting a server "
259 "(cooker) process.") 256 "(cooker) process.")
260 257
261 parser.add_option("", "--foreground", action="store_true",
262 help="Run bitbake server in foreground.")
263
264 parser.add_option("-B", "--bind", action="store", dest="bind", default=False, 258 parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
265 help="The name/address for the bitbake server to bind to.") 259 help="The name/address for the bitbake xmlrpc server to bind to.")
266 260
267 parser.add_option("-T", "--idle-timeout", type=int, 261 parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
268 default=int(os.environ.get("BBTIMEOUT", "0")), 262 default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None,
269 help="Set timeout to unload bitbake server due to inactivity") 263 help="Set timeout to unload bitbake server due to inactivity")
270 264
271 parser.add_option("", "--no-setscene", action="store_true", 265 parser.add_option("", "--no-setscene", action="store_true",
@@ -283,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
283 277
284 parser.add_option("-m", "--kill-server", action="store_true", 278 parser.add_option("-m", "--kill-server", action="store_true",
285 dest="kill_server", default=False, 279 dest="kill_server", default=False,
286 help="Terminate the remote server.") 280 help="Terminate the bitbake server.")
287 281
288 parser.add_option("", "--observe-only", action="store_true", 282 parser.add_option("", "--observe-only", action="store_true",
289 dest="observe_only", default=False, 283 dest="observe_only", default=False,
@@ -322,70 +316,20 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
322 eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S") 316 eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
323 options.writeeventlog = eventlog 317 options.writeeventlog = eventlog
324 318
325 # if BBSERVER says to autodetect, let's do that 319 if options.bind:
326 if options.remote_server: 320 try:
327 port = -1 321 #Checking that the port is a number and is a ':' delimited value
328 if options.remote_server != 'autostart': 322 (host, port) = options.bind.split(':')
329 host, port = options.remote_server.split(":", 2)
330 port = int(port) 323 port = int(port)
331 # use automatic port if port set to -1, means read it from 324 except (ValueError,IndexError):
332 # the bitbake.lock file; this is a bit tricky, but we always expect 325 raise BBMainException("FATAL: Malformed host:port bind parameter")
333 # to be in the base of the build directory if we need to have a 326 options.xmlrpcinterface = (host, port)
334 # chance to start the server later, anyway 327 else:
335 if port == -1: 328 options.xmlrpcinterface = (None, 0)
336 lock_location = "./bitbake.lock"
337 # we try to read the address at all times; if the server is not started,
338 # we'll try to start it after the first connect fails, below
339 try:
340 lf = open(lock_location, 'r')
341 remotedef = lf.readline()
342 [host, port] = remotedef.split(":")
343 port = int(port)
344 lf.close()
345 options.remote_server = remotedef
346 except Exception as e:
347 if options.remote_server != 'autostart':
348 raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
349 329
350 return options, targets[1:] 330 return options, targets[1:]
351 331
352 332
353def start_server(servermodule, configParams, configuration, features):
354 server = servermodule.BitBakeServer()
355 single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
356 if configParams.bind:
357 (host, port) = configParams.bind.split(':')
358 server.initServer((host, int(port)), single_use=single_use,
359 idle_timeout=configParams.idle_timeout)
360 configuration.interface = [server.serverImpl.host, server.serverImpl.port]
361 else:
362 server.initServer(single_use=single_use)
363 configuration.interface = []
364
365 try:
366 configuration.setServerRegIdleCallback(server.getServerIdleCB())
367
368 cooker = bb.cooker.BBCooker(configuration, features)
369
370 server.addcooker(cooker)
371 server.saveConnectionDetails()
372 except Exception as e:
373 while hasattr(server, "event_queue"):
374 import queue
375 try:
376 event = server.event_queue.get(block=False)
377 except (queue.Empty, IOError):
378 break
379 if isinstance(event, logging.LogRecord):
380 logger.handle(event)
381 raise
382 if not configParams.foreground:
383 server.detach()
384 cooker.shutdown()
385 cooker.lock.close()
386 return server
387
388
389def bitbake_main(configParams, configuration): 333def bitbake_main(configParams, configuration):
390 334
391 # Python multiprocessing requires /dev/shm on Linux 335 # Python multiprocessing requires /dev/shm on Linux
@@ -406,45 +350,15 @@ def bitbake_main(configParams, configuration):
406 350
407 configuration.setConfigParameters(configParams) 351 configuration.setConfigParameters(configParams)
408 352
409 if configParams.server_only: 353 if configParams.server_only and configParams.remote_server:
410 if configParams.servertype != "xmlrpc":
411 raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
412 "servertype as 'xmlrpc'.\n")
413 if not configParams.bind:
414 raise BBMainException("FATAL: The '--server-only' option requires a name/address "
415 "to bind to with the -B option.\n")
416 else:
417 try:
418 #Checking that the port is a number
419 int(configParams.bind.split(":")[1])
420 except (ValueError,IndexError):
421 raise BBMainException(
422 "FATAL: Malformed host:port bind parameter")
423 if configParams.remote_server:
424 raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % 354 raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
425 ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ 355 ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
426 else "the '--remote-server' option")) 356 else "the '--remote-server' option"))
427 357
428 elif configParams.foreground:
429 raise BBMainException("FATAL: The '--foreground' option can only be used "
430 "with --server-only.\n")
431
432 if configParams.bind and configParams.servertype != "xmlrpc":
433 raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
434 "set the servertype as 'xmlrpc'.\n")
435
436 if configParams.remote_server and configParams.servertype != "xmlrpc":
437 raise BBMainException("FATAL: If '--remote-server' is defined, we must "
438 "set the servertype as 'xmlrpc'.\n")
439
440 if configParams.observe_only and (not configParams.remote_server or configParams.bind): 358 if configParams.observe_only and (not configParams.remote_server or configParams.bind):
441 raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " 359 raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
442 "connecting to a server.\n") 360 "connecting to a server.\n")
443 361
444 if configParams.kill_server and not configParams.remote_server:
445 raise BBMainException("FATAL: '--kill-server' can only be used to "
446 "terminate a remote server")
447
448 if "BBDEBUG" in os.environ: 362 if "BBDEBUG" in os.environ:
449 level = int(os.environ["BBDEBUG"]) 363 level = int(os.environ["BBDEBUG"])
450 if level > configuration.debug: 364 if level > configuration.debug:
@@ -453,7 +367,7 @@ def bitbake_main(configParams, configuration):
453 bb.msg.init_msgconfig(configParams.verbose, configuration.debug, 367 bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
454 configuration.debug_domains) 368 configuration.debug_domains)
455 369
456 server, server_connection, ui_module = setup_bitbake(configParams, configuration) 370 server_connection, ui_module = setup_bitbake(configParams, configuration)
457 if server_connection is None and configParams.kill_server: 371 if server_connection is None and configParams.kill_server:
458 return 0 372 return 0
459 373
@@ -463,16 +377,15 @@ def bitbake_main(configParams, configuration):
463 return 0 377 return 0
464 378
465 try: 379 try:
380 for event in bb.event.ui_queue:
381 server_connection.events.queue_event(event)
382 bb.event.ui_queue = []
383
466 return ui_module.main(server_connection.connection, server_connection.events, 384 return ui_module.main(server_connection.connection, server_connection.events,
467 configParams) 385 configParams)
468 finally: 386 finally:
469 bb.event.ui_queue = []
470 server_connection.terminate() 387 server_connection.terminate()
471 else: 388 else:
472 print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
473 server.serverImpl.port))
474 if configParams.foreground:
475 server.serverImpl.serve_forever()
476 return 0 389 return 0
477 390
478 return 1 391 return 1
@@ -495,58 +408,69 @@ def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging
495 # Collect the feature set for the UI 408 # Collect the feature set for the UI
496 featureset = getattr(ui_module, "featureSet", []) 409 featureset = getattr(ui_module, "featureSet", [])
497 410
498 if configParams.server_only:
499 for param in ('prefile', 'postfile'):
500 value = getattr(configParams, param)
501 if value:
502 setattr(configuration, "%s_server" % param, value)
503 param = "%s_server" % param
504
505 if extrafeatures: 411 if extrafeatures:
506 for feature in extrafeatures: 412 for feature in extrafeatures:
507 if not feature in featureset: 413 if not feature in featureset:
508 featureset.append(feature) 414 featureset.append(feature)
509 415
510 servermodule = import_extension_module(bb.server, 416 server_connection = None
511 configParams.servertype, 417
512 'BitBakeServer')
513 if configParams.remote_server: 418 if configParams.remote_server:
514 if os.getenv('BBSERVER') == 'autostart': 419 # Connect to a remote XMLRPC server
515 if configParams.remote_server == 'autostart' or \ 420 server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
516 not servermodule.check_connection(configParams.remote_server, timeout=2): 421 configParams.observe_only, configParams.xmlrpctoken)
517 configParams.bind = 'localhost:0'
518 srv = start_server(servermodule, configParams, configuration, featureset)
519 configParams.remote_server = '%s:%d' % tuple(configuration.interface)
520 bb.event.ui_queue = []
521 # we start a stub server that is actually a XMLRPClient that connects to a real server
522 from bb.server.xmlrpc import BitBakeXMLRPCClient
523 server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
524 configParams.xmlrpctoken)
525 server.saveConnectionDetails(configParams.remote_server)
526 else: 422 else:
527 # we start a server with a given configuration 423 retries = 8
528 server = start_server(servermodule, configParams, configuration, featureset) 424 while retries:
425 try:
426 topdir, lock = lockBitbake()
427 sockname = topdir + "/bitbake.sock"
428 if lock:
429 # we start a server with a given configuration
430 logger.info("Starting bitbake server...")
431 server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
432 # The server will handle any events already in the queue
433 bb.event.ui_queue = []
434 else:
435 logger.info("Reconnecting to bitbake server...")
436 if not os.path.exists(sockname):
437 print("Previous bitbake instance shutting down?, waiting to retry...")
438 time.sleep(5)
439 raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
440 if not configParams.server_only:
441 server_connection = bb.server.process.connectProcessServer(sockname, featureset)
442 if server_connection:
443 break
444 except (Exception, bb.server.process.ProcessTimeout) as e:
445 if not retries:
446 raise
447 retries -= 1
448 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
449 logger.info("Retrying server connection...")
450 else:
451 logger.info("Retrying server connection... (%s)" % traceback.format_exc())
452 if not retries:
453 bb.fatal("Unable to connect to bitbake server, or start one")
454 if retries < 5:
455 time.sleep(5)
456
457 if configParams.kill_server:
458 server_connection.connection.terminateServer()
459 server_connection.terminate()
529 bb.event.ui_queue = [] 460 bb.event.ui_queue = []
461 logger.info("Terminated bitbake server.")
462 return None, None
530 463
531 if configParams.server_only: 464 # Restore the environment in case the UI needs it
532 server_connection = None 465 for k in cleanedvars:
533 else: 466 os.environ[k] = cleanedvars[k]
534 try:
535 server_connection = server.establishConnection(featureset)
536 except Exception as e:
537 bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
538
539 if configParams.kill_server:
540 server_connection.connection.terminateServer()
541 bb.event.ui_queue = []
542 return None, None, None
543 467
544 server_connection.setupEventQueue() 468 logger.removeHandler(handler)
545 469
546 # Restore the environment in case the UI needs it 470 return server_connection, ui_module
547 for k in cleanedvars:
548 os.environ[k] = cleanedvars[k]
549 471
550 logger.removeHandler(handler) 472def lockBitbake():
473 topdir = bb.cookerdata.findTopdir()
474 lockfile = topdir + "/bitbake.lock"
475 return topdir, bb.utils.lockfile(lockfile, False, False)
551 476
552 return server, server_connection, ui_module