diff options
-rw-r--r-- | bitbake/lib/bb/cooker.py | 58 | ||||
-rw-r--r-- | bitbake/lib/bb/cookerdata.py | 5 | ||||
-rwxr-xr-x | bitbake/lib/bb/main.py | 230 | ||||
-rw-r--r-- | bitbake/lib/bb/server/__init__.py | 6 | ||||
-rw-r--r-- | bitbake/lib/bb/server/process.py | 596 | ||||
-rw-r--r-- | bitbake/lib/bb/server/xmlrpc.py | 492 | ||||
-rw-r--r-- | bitbake/lib/bb/server/xmlrpcclient.py | 154 | ||||
-rw-r--r-- | bitbake/lib/bb/server/xmlrpcserver.py | 158 | ||||
-rw-r--r-- | bitbake/lib/bb/tinfoil.py | 2 | ||||
-rw-r--r-- | bitbake/lib/prserv/serv.py | 4 |
10 files changed, 789 insertions, 916 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index b1311bb170..e27763ecab 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -215,19 +215,6 @@ class BBCooker: | |||
215 | 215 | ||
216 | self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier]) | 216 | self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier]) |
217 | 217 | ||
218 | # Take a lock so only one copy of bitbake can run against a given build | ||
219 | # directory at a time | ||
220 | if not self.lockBitbake(): | ||
221 | bb.fatal("Only one copy of bitbake should be run against a build directory") | ||
222 | try: | ||
223 | self.lock.seek(0) | ||
224 | self.lock.truncate() | ||
225 | if len(configuration.interface) >= 2: | ||
226 | self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1])); | ||
227 | self.lock.flush() | ||
228 | except: | ||
229 | pass | ||
230 | |||
231 | # TOSTOP must not be set or our children will hang when they output | 218 | # TOSTOP must not be set or our children will hang when they output |
232 | try: | 219 | try: |
233 | fd = sys.stdout.fileno() | 220 | fd = sys.stdout.fileno() |
@@ -1557,33 +1544,6 @@ class BBCooker: | |||
1557 | def post_serve(self): | 1544 | def post_serve(self): |
1558 | prserv.serv.auto_shutdown(self.data) | 1545 | prserv.serv.auto_shutdown(self.data) |
1559 | bb.event.fire(CookerExit(), self.data) | 1546 | bb.event.fire(CookerExit(), self.data) |
1560 | lockfile = self.lock.name | ||
1561 | self.lock.close() | ||
1562 | self.lock = None | ||
1563 | |||
1564 | while not self.lock: | ||
1565 | with bb.utils.timeout(3): | ||
1566 | self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) | ||
1567 | if not self.lock: | ||
1568 | # Some systems may not have lsof available | ||
1569 | procs = None | ||
1570 | try: | ||
1571 | procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) | ||
1572 | except OSError as e: | ||
1573 | if e.errno != errno.ENOENT: | ||
1574 | raise | ||
1575 | if procs is None: | ||
1576 | # Fall back to fuser if lsof is unavailable | ||
1577 | try: | ||
1578 | procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) | ||
1579 | except OSError as e: | ||
1580 | if e.errno != errno.ENOENT: | ||
1581 | raise | ||
1582 | |||
1583 | msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" | ||
1584 | if procs: | ||
1585 | msg += ":\n%s" % str(procs) | ||
1586 | print(msg) | ||
1587 | 1547 | ||
1588 | 1548 | ||
1589 | def shutdown(self, force = False): | 1549 | def shutdown(self, force = False): |
@@ -1605,23 +1565,7 @@ class BBCooker: | |||
1605 | 1565 | ||
1606 | def clientComplete(self): | 1566 | def clientComplete(self): |
1607 | """Called when the client is done using the server""" | 1567 | """Called when the client is done using the server""" |
1608 | if self.configuration.server_only: | 1568 | self.finishcommand() |
1609 | self.finishcommand() | ||
1610 | else: | ||
1611 | self.shutdown(True) | ||
1612 | |||
1613 | def lockBitbake(self): | ||
1614 | if not hasattr(self, 'lock'): | ||
1615 | self.lock = None | ||
1616 | if self.data: | ||
1617 | lockfile = self.data.expand("${TOPDIR}/bitbake.lock") | ||
1618 | if lockfile: | ||
1619 | self.lock = bb.utils.lockfile(lockfile, False, False) | ||
1620 | return self.lock | ||
1621 | |||
1622 | def unlockBitbake(self): | ||
1623 | if hasattr(self, 'lock') and self.lock: | ||
1624 | bb.utils.unlockfile(self.lock) | ||
1625 | 1569 | ||
1626 | def server_main(cooker, func, *args): | 1570 | def server_main(cooker, func, *args): |
1627 | cooker.pre_serve() | 1571 | cooker.pre_serve() |
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py index 6511dcbfad..d05abfe745 100644 --- a/bitbake/lib/bb/cookerdata.py +++ b/bitbake/lib/bb/cookerdata.py | |||
@@ -76,7 +76,7 @@ class ConfigParameters(object): | |||
76 | for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", | 76 | for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", |
77 | "verbose", "debug", "dry_run", "dump_signatures", | 77 | "verbose", "debug", "dry_run", "dump_signatures", |
78 | "debug_domains", "extra_assume_provided", "profile", | 78 | "debug_domains", "extra_assume_provided", "profile", |
79 | "prefile", "postfile", "tracking"]: | 79 | "prefile", "postfile", "tracking", "server_timeout"]: |
80 | options[o] = getattr(self.options, o) | 80 | options[o] = getattr(self.options, o) |
81 | 81 | ||
82 | ret, error = server.runCommand(["updateConfig", options, environment, sys.argv]) | 82 | ret, error = server.runCommand(["updateConfig", options, environment, sys.argv]) |
@@ -144,7 +144,8 @@ class CookerConfiguration(object): | |||
144 | self.dump_signatures = [] | 144 | self.dump_signatures = [] |
145 | self.dry_run = False | 145 | self.dry_run = False |
146 | self.tracking = False | 146 | self.tracking = False |
147 | self.interface = [] | 147 | self.xmlrpcinterface = [] |
148 | self.server_timeout = None | ||
148 | self.writeeventlog = False | 149 | self.writeeventlog = False |
149 | self.server_only = False | 150 | self.server_only = False |
150 | self.limited_deps = False | 151 | self.limited_deps = False |
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py index 29e391162e..1edf56f41b 100755 --- a/bitbake/lib/bb/main.py +++ b/bitbake/lib/bb/main.py | |||
@@ -28,6 +28,8 @@ import logging | |||
28 | import optparse | 28 | import optparse |
29 | import warnings | 29 | import warnings |
30 | import fcntl | 30 | import fcntl |
31 | import time | ||
32 | import traceback | ||
31 | 33 | ||
32 | import bb | 34 | import bb |
33 | from bb import event | 35 | from bb import event |
@@ -37,6 +39,9 @@ from bb import ui | |||
37 | from bb import server | 39 | from bb import server |
38 | from bb import cookerdata | 40 | from bb import cookerdata |
39 | 41 | ||
42 | import bb.server.process | ||
43 | import bb.server.xmlrpcclient | ||
44 | |||
40 | logger = logging.getLogger("BitBake") | 45 | logger = logging.getLogger("BitBake") |
41 | 46 | ||
42 | class BBMainException(Exception): | 47 | class BBMainException(Exception): |
@@ -58,9 +63,6 @@ class BitbakeHelpFormatter(optparse.IndentedHelpFormatter): | |||
58 | if option.dest == 'ui': | 63 | if option.dest == 'ui': |
59 | valid_uis = list_extension_modules(bb.ui, 'main') | 64 | valid_uis = list_extension_modules(bb.ui, 'main') |
60 | option.help = option.help.replace('@CHOICES@', present_options(valid_uis)) | 65 | option.help = option.help.replace('@CHOICES@', present_options(valid_uis)) |
61 | elif option.dest == 'servertype': | ||
62 | valid_server_types = list_extension_modules(bb.server, 'BitBakeServer') | ||
63 | option.help = option.help.replace('@CHOICES@', present_options(valid_server_types)) | ||
64 | 66 | ||
65 | return optparse.IndentedHelpFormatter.format_option(self, option) | 67 | return optparse.IndentedHelpFormatter.format_option(self, option) |
66 | 68 | ||
@@ -238,11 +240,6 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): | |||
238 | default=os.environ.get('BITBAKE_UI', 'knotty'), | 240 | default=os.environ.get('BITBAKE_UI', 'knotty'), |
239 | help="The user interface to use (@CHOICES@ - default %default).") | 241 | help="The user interface to use (@CHOICES@ - default %default).") |
240 | 242 | ||
241 | # @CHOICES@ is substituted out by BitbakeHelpFormatter above | ||
242 | parser.add_option("-t", "--servertype", action="store", dest="servertype", | ||
243 | default=["process", "xmlrpc"]["BBSERVER" in os.environ], | ||
244 | help="Choose which server type to use (@CHOICES@ - default %default).") | ||
245 | |||
246 | parser.add_option("", "--token", action="store", dest="xmlrpctoken", | 243 | parser.add_option("", "--token", action="store", dest="xmlrpctoken", |
247 | default=os.environ.get("BBTOKEN"), | 244 | default=os.environ.get("BBTOKEN"), |
248 | help="Specify the connection token to be used when connecting " | 245 | help="Specify the connection token to be used when connecting " |
@@ -258,14 +255,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): | |||
258 | help="Run bitbake without a UI, only starting a server " | 255 | help="Run bitbake without a UI, only starting a server " |
259 | "(cooker) process.") | 256 | "(cooker) process.") |
260 | 257 | ||
261 | parser.add_option("", "--foreground", action="store_true", | ||
262 | help="Run bitbake server in foreground.") | ||
263 | |||
264 | parser.add_option("-B", "--bind", action="store", dest="bind", default=False, | 258 | parser.add_option("-B", "--bind", action="store", dest="bind", default=False, |
265 | help="The name/address for the bitbake server to bind to.") | 259 | help="The name/address for the bitbake xmlrpc server to bind to.") |
266 | 260 | ||
267 | parser.add_option("-T", "--idle-timeout", type=int, | 261 | parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout", |
268 | default=int(os.environ.get("BBTIMEOUT", "0")), | 262 | default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None, |
269 | help="Set timeout to unload bitbake server due to inactivity") | 263 | help="Set timeout to unload bitbake server due to inactivity") |
270 | 264 | ||
271 | parser.add_option("", "--no-setscene", action="store_true", | 265 | parser.add_option("", "--no-setscene", action="store_true", |
@@ -283,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): | |||
283 | 277 | ||
284 | parser.add_option("-m", "--kill-server", action="store_true", | 278 | parser.add_option("-m", "--kill-server", action="store_true", |
285 | dest="kill_server", default=False, | 279 | dest="kill_server", default=False, |
286 | help="Terminate the remote server.") | 280 | help="Terminate the bitbake server.") |
287 | 281 | ||
288 | parser.add_option("", "--observe-only", action="store_true", | 282 | parser.add_option("", "--observe-only", action="store_true", |
289 | dest="observe_only", default=False, | 283 | dest="observe_only", default=False, |
@@ -322,70 +316,20 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): | |||
322 | eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S") | 316 | eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S") |
323 | options.writeeventlog = eventlog | 317 | options.writeeventlog = eventlog |
324 | 318 | ||
325 | # if BBSERVER says to autodetect, let's do that | 319 | if options.bind: |
326 | if options.remote_server: | 320 | try: |
327 | port = -1 | 321 | #Checking that the port is a number and is a ':' delimited value |
328 | if options.remote_server != 'autostart': | 322 | (host, port) = options.bind.split(':') |
329 | host, port = options.remote_server.split(":", 2) | ||
330 | port = int(port) | 323 | port = int(port) |
331 | # use automatic port if port set to -1, means read it from | 324 | except (ValueError,IndexError): |
332 | # the bitbake.lock file; this is a bit tricky, but we always expect | 325 | raise BBMainException("FATAL: Malformed host:port bind parameter") |
333 | # to be in the base of the build directory if we need to have a | 326 | options.xmlrpcinterface = (host, port) |
334 | # chance to start the server later, anyway | 327 | else: |
335 | if port == -1: | 328 | options.xmlrpcinterface = (None, 0) |
336 | lock_location = "./bitbake.lock" | ||
337 | # we try to read the address at all times; if the server is not started, | ||
338 | # we'll try to start it after the first connect fails, below | ||
339 | try: | ||
340 | lf = open(lock_location, 'r') | ||
341 | remotedef = lf.readline() | ||
342 | [host, port] = remotedef.split(":") | ||
343 | port = int(port) | ||
344 | lf.close() | ||
345 | options.remote_server = remotedef | ||
346 | except Exception as e: | ||
347 | if options.remote_server != 'autostart': | ||
348 | raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e)) | ||
349 | 329 | ||
350 | return options, targets[1:] | 330 | return options, targets[1:] |
351 | 331 | ||
352 | 332 | ||
353 | def start_server(servermodule, configParams, configuration, features): | ||
354 | server = servermodule.BitBakeServer() | ||
355 | single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart' | ||
356 | if configParams.bind: | ||
357 | (host, port) = configParams.bind.split(':') | ||
358 | server.initServer((host, int(port)), single_use=single_use, | ||
359 | idle_timeout=configParams.idle_timeout) | ||
360 | configuration.interface = [server.serverImpl.host, server.serverImpl.port] | ||
361 | else: | ||
362 | server.initServer(single_use=single_use) | ||
363 | configuration.interface = [] | ||
364 | |||
365 | try: | ||
366 | configuration.setServerRegIdleCallback(server.getServerIdleCB()) | ||
367 | |||
368 | cooker = bb.cooker.BBCooker(configuration, features) | ||
369 | |||
370 | server.addcooker(cooker) | ||
371 | server.saveConnectionDetails() | ||
372 | except Exception as e: | ||
373 | while hasattr(server, "event_queue"): | ||
374 | import queue | ||
375 | try: | ||
376 | event = server.event_queue.get(block=False) | ||
377 | except (queue.Empty, IOError): | ||
378 | break | ||
379 | if isinstance(event, logging.LogRecord): | ||
380 | logger.handle(event) | ||
381 | raise | ||
382 | if not configParams.foreground: | ||
383 | server.detach() | ||
384 | cooker.shutdown() | ||
385 | cooker.lock.close() | ||
386 | return server | ||
387 | |||
388 | |||
389 | def bitbake_main(configParams, configuration): | 333 | def bitbake_main(configParams, configuration): |
390 | 334 | ||
391 | # Python multiprocessing requires /dev/shm on Linux | 335 | # Python multiprocessing requires /dev/shm on Linux |
@@ -406,45 +350,15 @@ def bitbake_main(configParams, configuration): | |||
406 | 350 | ||
407 | configuration.setConfigParameters(configParams) | 351 | configuration.setConfigParameters(configParams) |
408 | 352 | ||
409 | if configParams.server_only: | 353 | if configParams.server_only and configParams.remote_server: |
410 | if configParams.servertype != "xmlrpc": | ||
411 | raise BBMainException("FATAL: If '--server-only' is defined, we must set the " | ||
412 | "servertype as 'xmlrpc'.\n") | ||
413 | if not configParams.bind: | ||
414 | raise BBMainException("FATAL: The '--server-only' option requires a name/address " | ||
415 | "to bind to with the -B option.\n") | ||
416 | else: | ||
417 | try: | ||
418 | #Checking that the port is a number | ||
419 | int(configParams.bind.split(":")[1]) | ||
420 | except (ValueError,IndexError): | ||
421 | raise BBMainException( | ||
422 | "FATAL: Malformed host:port bind parameter") | ||
423 | if configParams.remote_server: | ||
424 | raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % | 354 | raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % |
425 | ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ | 355 | ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ |
426 | else "the '--remote-server' option")) | 356 | else "the '--remote-server' option")) |
427 | 357 | ||
428 | elif configParams.foreground: | ||
429 | raise BBMainException("FATAL: The '--foreground' option can only be used " | ||
430 | "with --server-only.\n") | ||
431 | |||
432 | if configParams.bind and configParams.servertype != "xmlrpc": | ||
433 | raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must " | ||
434 | "set the servertype as 'xmlrpc'.\n") | ||
435 | |||
436 | if configParams.remote_server and configParams.servertype != "xmlrpc": | ||
437 | raise BBMainException("FATAL: If '--remote-server' is defined, we must " | ||
438 | "set the servertype as 'xmlrpc'.\n") | ||
439 | |||
440 | if configParams.observe_only and (not configParams.remote_server or configParams.bind): | 358 | if configParams.observe_only and (not configParams.remote_server or configParams.bind): |
441 | raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " | 359 | raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " |
442 | "connecting to a server.\n") | 360 | "connecting to a server.\n") |
443 | 361 | ||
444 | if configParams.kill_server and not configParams.remote_server: | ||
445 | raise BBMainException("FATAL: '--kill-server' can only be used to " | ||
446 | "terminate a remote server") | ||
447 | |||
448 | if "BBDEBUG" in os.environ: | 362 | if "BBDEBUG" in os.environ: |
449 | level = int(os.environ["BBDEBUG"]) | 363 | level = int(os.environ["BBDEBUG"]) |
450 | if level > configuration.debug: | 364 | if level > configuration.debug: |
@@ -453,7 +367,7 @@ def bitbake_main(configParams, configuration): | |||
453 | bb.msg.init_msgconfig(configParams.verbose, configuration.debug, | 367 | bb.msg.init_msgconfig(configParams.verbose, configuration.debug, |
454 | configuration.debug_domains) | 368 | configuration.debug_domains) |
455 | 369 | ||
456 | server, server_connection, ui_module = setup_bitbake(configParams, configuration) | 370 | server_connection, ui_module = setup_bitbake(configParams, configuration) |
457 | if server_connection is None and configParams.kill_server: | 371 | if server_connection is None and configParams.kill_server: |
458 | return 0 | 372 | return 0 |
459 | 373 | ||
@@ -463,16 +377,15 @@ def bitbake_main(configParams, configuration): | |||
463 | return 0 | 377 | return 0 |
464 | 378 | ||
465 | try: | 379 | try: |
380 | for event in bb.event.ui_queue: | ||
381 | server_connection.events.queue_event(event) | ||
382 | bb.event.ui_queue = [] | ||
383 | |||
466 | return ui_module.main(server_connection.connection, server_connection.events, | 384 | return ui_module.main(server_connection.connection, server_connection.events, |
467 | configParams) | 385 | configParams) |
468 | finally: | 386 | finally: |
469 | bb.event.ui_queue = [] | ||
470 | server_connection.terminate() | 387 | server_connection.terminate() |
471 | else: | 388 | else: |
472 | print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host, | ||
473 | server.serverImpl.port)) | ||
474 | if configParams.foreground: | ||
475 | server.serverImpl.serve_forever() | ||
476 | return 0 | 389 | return 0 |
477 | 390 | ||
478 | return 1 | 391 | return 1 |
@@ -495,58 +408,69 @@ def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging | |||
495 | # Collect the feature set for the UI | 408 | # Collect the feature set for the UI |
496 | featureset = getattr(ui_module, "featureSet", []) | 409 | featureset = getattr(ui_module, "featureSet", []) |
497 | 410 | ||
498 | if configParams.server_only: | ||
499 | for param in ('prefile', 'postfile'): | ||
500 | value = getattr(configParams, param) | ||
501 | if value: | ||
502 | setattr(configuration, "%s_server" % param, value) | ||
503 | param = "%s_server" % param | ||
504 | |||
505 | if extrafeatures: | 411 | if extrafeatures: |
506 | for feature in extrafeatures: | 412 | for feature in extrafeatures: |
507 | if not feature in featureset: | 413 | if not feature in featureset: |
508 | featureset.append(feature) | 414 | featureset.append(feature) |
509 | 415 | ||
510 | servermodule = import_extension_module(bb.server, | 416 | server_connection = None |
511 | configParams.servertype, | 417 | |
512 | 'BitBakeServer') | ||
513 | if configParams.remote_server: | 418 | if configParams.remote_server: |
514 | if os.getenv('BBSERVER') == 'autostart': | 419 | # Connect to a remote XMLRPC server |
515 | if configParams.remote_server == 'autostart' or \ | 420 | server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset, |
516 | not servermodule.check_connection(configParams.remote_server, timeout=2): | 421 | configParams.observe_only, configParams.xmlrpctoken) |
517 | configParams.bind = 'localhost:0' | ||
518 | srv = start_server(servermodule, configParams, configuration, featureset) | ||
519 | configParams.remote_server = '%s:%d' % tuple(configuration.interface) | ||
520 | bb.event.ui_queue = [] | ||
521 | # we start a stub server that is actually a XMLRPClient that connects to a real server | ||
522 | from bb.server.xmlrpc import BitBakeXMLRPCClient | ||
523 | server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, | ||
524 | configParams.xmlrpctoken) | ||
525 | server.saveConnectionDetails(configParams.remote_server) | ||
526 | else: | 422 | else: |
527 | # we start a server with a given configuration | 423 | retries = 8 |
528 | server = start_server(servermodule, configParams, configuration, featureset) | 424 | while retries: |
425 | try: | ||
426 | topdir, lock = lockBitbake() | ||
427 | sockname = topdir + "/bitbake.sock" | ||
428 | if lock: | ||
429 | # we start a server with a given configuration | ||
430 | logger.info("Starting bitbake server...") | ||
431 | server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) | ||
432 | # The server will handle any events already in the queue | ||
433 | bb.event.ui_queue = [] | ||
434 | else: | ||
435 | logger.info("Reconnecting to bitbake server...") | ||
436 | if not os.path.exists(sockname): | ||
437 | print("Previous bitbake instance shutting down?, waiting to retry...") | ||
438 | time.sleep(5) | ||
439 | raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?") | ||
440 | if not configParams.server_only: | ||
441 | server_connection = bb.server.process.connectProcessServer(sockname, featureset) | ||
442 | if server_connection: | ||
443 | break | ||
444 | except (Exception, bb.server.process.ProcessTimeout) as e: | ||
445 | if not retries: | ||
446 | raise | ||
447 | retries -= 1 | ||
448 | if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)): | ||
449 | logger.info("Retrying server connection...") | ||
450 | else: | ||
451 | logger.info("Retrying server connection... (%s)" % traceback.format_exc()) | ||
452 | if not retries: | ||
453 | bb.fatal("Unable to connect to bitbake server, or start one") | ||
454 | if retries < 5: | ||
455 | time.sleep(5) | ||
456 | |||
457 | if configParams.kill_server: | ||
458 | server_connection.connection.terminateServer() | ||
459 | server_connection.terminate() | ||
529 | bb.event.ui_queue = [] | 460 | bb.event.ui_queue = [] |
461 | logger.info("Terminated bitbake server.") | ||
462 | return None, None | ||
530 | 463 | ||
531 | if configParams.server_only: | 464 | # Restore the environment in case the UI needs it |
532 | server_connection = None | 465 | for k in cleanedvars: |
533 | else: | 466 | os.environ[k] = cleanedvars[k] |
534 | try: | ||
535 | server_connection = server.establishConnection(featureset) | ||
536 | except Exception as e: | ||
537 | bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e))) | ||
538 | |||
539 | if configParams.kill_server: | ||
540 | server_connection.connection.terminateServer() | ||
541 | bb.event.ui_queue = [] | ||
542 | return None, None, None | ||
543 | 467 | ||
544 | server_connection.setupEventQueue() | 468 | logger.removeHandler(handler) |
545 | 469 | ||
546 | # Restore the environment in case the UI needs it | 470 | return server_connection, ui_module |
547 | for k in cleanedvars: | ||
548 | os.environ[k] = cleanedvars[k] | ||
549 | 471 | ||
550 | logger.removeHandler(handler) | 472 | def lockBitbake(): |
473 | topdir = bb.cookerdata.findTopdir() | ||
474 | lockfile = topdir + "/bitbake.lock" | ||
475 | return topdir, bb.utils.lockfile(lockfile, False, False) | ||
551 | 476 | ||
552 | return server, server_connection, ui_module | ||
diff --git a/bitbake/lib/bb/server/__init__.py b/bitbake/lib/bb/server/__init__.py index 345691e40f..5a3fba968f 100644 --- a/bitbake/lib/bb/server/__init__.py +++ b/bitbake/lib/bb/server/__init__.py | |||
@@ -18,10 +18,4 @@ | |||
18 | # with this program; if not, write to the Free Software Foundation, Inc., | 18 | # with this program; if not, write to the Free Software Foundation, Inc., |
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
20 | 20 | ||
21 | """ Base code for Bitbake server process | ||
22 | |||
23 | Have a common base for that all Bitbake server classes ensures a consistent | ||
24 | approach to the interface, and minimize risks associated with code duplication. | ||
25 | |||
26 | """ | ||
27 | 21 | ||
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index 48da7fe46c..6edb0213ad 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py | |||
@@ -22,128 +22,205 @@ | |||
22 | 22 | ||
23 | import bb | 23 | import bb |
24 | import bb.event | 24 | import bb.event |
25 | import itertools | ||
26 | import logging | 25 | import logging |
27 | import multiprocessing | 26 | import multiprocessing |
27 | import threading | ||
28 | import array | ||
28 | import os | 29 | import os |
29 | import signal | ||
30 | import sys | 30 | import sys |
31 | import time | 31 | import time |
32 | import select | 32 | import select |
33 | from queue import Empty | 33 | import socket |
34 | from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager | 34 | import subprocess |
35 | import errno | ||
36 | import bb.server.xmlrpcserver | ||
37 | from bb import daemonize | ||
38 | from multiprocessing import queues | ||
35 | 39 | ||
36 | logger = logging.getLogger('BitBake') | 40 | logger = logging.getLogger('BitBake') |
37 | 41 | ||
38 | class ServerCommunicator(): | 42 | class ProcessTimeout(SystemExit): |
39 | def __init__(self, connection, event_handle, server): | 43 | pass |
40 | self.connection = connection | ||
41 | self.event_handle = event_handle | ||
42 | self.server = server | ||
43 | |||
44 | def runCommand(self, command): | ||
45 | # @todo try/except | ||
46 | self.connection.send(command) | ||
47 | |||
48 | if not self.server.is_alive(): | ||
49 | raise SystemExit | ||
50 | |||
51 | while True: | ||
52 | # don't let the user ctrl-c while we're waiting for a response | ||
53 | try: | ||
54 | for idx in range(0,4): # 0, 1, 2, 3 | ||
55 | if self.connection.poll(5): | ||
56 | return self.connection.recv() | ||
57 | else: | ||
58 | bb.warn("Timeout while attempting to communicate with bitbake server") | ||
59 | bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server") | ||
60 | except KeyboardInterrupt: | ||
61 | pass | ||
62 | |||
63 | def getEventHandle(self): | ||
64 | handle, error = self.runCommand(["getUIHandlerNum"]) | ||
65 | if error: | ||
66 | logger.error("Unable to get UI Handler Number: %s" % error) | ||
67 | raise BaseException(error) | ||
68 | 44 | ||
69 | return handle | 45 | class ProcessServer(multiprocessing.Process): |
70 | |||
71 | class EventAdapter(): | ||
72 | """ | ||
73 | Adapter to wrap our event queue since the caller (bb.event) expects to | ||
74 | call a send() method, but our actual queue only has put() | ||
75 | """ | ||
76 | def __init__(self, queue): | ||
77 | self.queue = queue | ||
78 | |||
79 | def send(self, event): | ||
80 | try: | ||
81 | self.queue.put(event) | ||
82 | except Exception as err: | ||
83 | print("EventAdapter puked: %s" % str(err)) | ||
84 | |||
85 | |||
86 | class ProcessServer(Process): | ||
87 | profile_filename = "profile.log" | 46 | profile_filename = "profile.log" |
88 | profile_processed_filename = "profile.log.processed" | 47 | profile_processed_filename = "profile.log.processed" |
89 | 48 | ||
90 | def __init__(self, command_channel, event_queue, featurelist): | 49 | def __init__(self, lock, sock, sockname): |
91 | self._idlefuns = {} | 50 | multiprocessing.Process.__init__(self) |
92 | Process.__init__(self) | 51 | self.command_channel = False |
93 | self.command_channel = command_channel | 52 | self.command_channel_reply = False |
94 | self.event_queue = event_queue | ||
95 | self.event = EventAdapter(event_queue) | ||
96 | self.featurelist = featurelist | ||
97 | self.quit = False | 53 | self.quit = False |
98 | self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore. | 54 | self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore. |
99 | self.next_heartbeat = time.time() | 55 | self.next_heartbeat = time.time() |
100 | 56 | ||
101 | self.quitin, self.quitout = Pipe() | 57 | self.event_handle = None |
102 | self.event_handle = multiprocessing.Value("i") | 58 | self.haveui = False |
59 | self.lastui = False | ||
60 | self.xmlrpc = False | ||
61 | |||
62 | self._idlefuns = {} | ||
63 | |||
64 | self.bitbake_lock = lock | ||
65 | self.sock = sock | ||
66 | self.sockname = sockname | ||
67 | |||
68 | def register_idle_function(self, function, data): | ||
69 | """Register a function to be called while the server is idle""" | ||
70 | assert hasattr(function, '__call__') | ||
71 | self._idlefuns[function] = data | ||
103 | 72 | ||
104 | def run(self): | 73 | def run(self): |
105 | for event in bb.event.ui_queue: | 74 | |
106 | self.event_queue.put(event) | 75 | if self.xmlrpcinterface[0]: |
107 | self.event_handle.value = bb.event.register_UIHhandler(self, True) | 76 | self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self) |
77 | |||
78 | print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port)) | ||
108 | 79 | ||
109 | heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') | 80 | heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') |
110 | if heartbeat_event: | 81 | if heartbeat_event: |
111 | try: | 82 | try: |
112 | self.heartbeat_seconds = float(heartbeat_event) | 83 | self.heartbeat_seconds = float(heartbeat_event) |
113 | except: | 84 | except: |
114 | # Throwing an exception here causes bitbake to hang. | ||
115 | # Just warn about the invalid setting and continue | ||
116 | bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) | 85 | bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) |
86 | |||
87 | self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT') | ||
88 | try: | ||
89 | if self.timeout: | ||
90 | self.timeout = float(self.timeout) | ||
91 | except: | ||
92 | bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout) | ||
93 | |||
94 | |||
95 | try: | ||
96 | self.bitbake_lock.seek(0) | ||
97 | self.bitbake_lock.truncate() | ||
98 | if self.xmlrpcinterface[0]: | ||
99 | self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), configuration.interface[0], configuration.interface[1])) | ||
100 | else: | ||
101 | self.bitbake_lock.write("%s\n" % (os.getpid())) | ||
102 | self.bitbake_lock.flush() | ||
103 | except: | ||
104 | pass | ||
105 | |||
117 | bb.cooker.server_main(self.cooker, self.main) | 106 | bb.cooker.server_main(self.cooker, self.main) |
118 | 107 | ||
119 | def main(self): | 108 | def main(self): |
120 | # Ignore SIGINT within the server, as all SIGINT handling is done by | ||
121 | # the UI and communicated to us | ||
122 | self.quitin.close() | ||
123 | signal.signal(signal.SIGINT, signal.SIG_IGN) | ||
124 | bb.utils.set_process_name("Cooker") | 109 | bb.utils.set_process_name("Cooker") |
110 | |||
111 | ready = [] | ||
112 | |||
113 | self.controllersock = False | ||
114 | fds = [self.sock] | ||
115 | if self.xmlrpc: | ||
116 | fds.append(self.xmlrpc) | ||
125 | while not self.quit: | 117 | while not self.quit: |
126 | try: | 118 | if self.command_channel in ready: |
127 | if self.command_channel.poll(): | 119 | command = self.command_channel.get() |
128 | command = self.command_channel.recv() | 120 | if command[0] == "terminateServer": |
129 | self.runCommand(command) | ||
130 | if self.quitout.poll(): | ||
131 | self.quitout.recv() | ||
132 | self.quit = True | 121 | self.quit = True |
133 | try: | 122 | continue |
134 | self.runCommand(["stateForceShutdown"]) | 123 | try: |
135 | except: | 124 | print("Running command %s" % command) |
136 | pass | 125 | self.command_channel_reply.send(self.cooker.command.runCommand(command)) |
137 | 126 | except Exception as e: | |
138 | self.idle_commands(.1, [self.command_channel, self.quitout]) | 127 | logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e))) |
139 | except Exception: | 128 | |
140 | logger.exception('Running command %s', command) | 129 | if self.xmlrpc in ready: |
130 | self.xmlrpc.handle_requests() | ||
131 | if self.sock in ready: | ||
132 | self.controllersock, address = self.sock.accept() | ||
133 | if self.haveui: | ||
134 | print("Dropping connection attempt as we have a UI %s" % (str(ready))) | ||
135 | self.controllersock.close() | ||
136 | else: | ||
137 | print("Accepting %s" % (str(ready))) | ||
138 | fds.append(self.controllersock) | ||
139 | if self.controllersock in ready: | ||
140 | try: | ||
141 | print("Connecting Client") | ||
142 | ui_fds = recvfds(self.controllersock, 3) | ||
143 | |||
144 | # Where to write events to | ||
145 | writer = ConnectionWriter(ui_fds[0]) | ||
146 | self.event_handle = bb.event.register_UIHhandler(writer, True) | ||
147 | self.event_writer = writer | ||
148 | |||
149 | # Where to read commands from | ||
150 | reader = ConnectionReader(ui_fds[1]) | ||
151 | fds.append(reader) | ||
152 | self.command_channel = reader | ||
153 | |||
154 | # Where to send command return values to | ||
155 | writer = ConnectionWriter(ui_fds[2]) | ||
156 | self.command_channel_reply = writer | ||
157 | |||
158 | self.haveui = True | ||
159 | |||
160 | except EOFError: | ||
161 | print("Disconnecting Client") | ||
162 | fds.remove(self.controllersock) | ||
163 | fds.remove(self.command_channel) | ||
164 | bb.event.unregister_UIHhandler(self.event_handle, True) | ||
165 | self.command_channel_reply.writer.close() | ||
166 | self.event_writer.writer.close() | ||
167 | del self.event_writer | ||
168 | self.controllersock.close() | ||
169 | self.haveui = False | ||
170 | self.lastui = time.time() | ||
171 | self.cooker.clientComplete() | ||
172 | if self.timeout is None: | ||
173 | print("No timeout, exiting.") | ||
174 | self.quit = True | ||
175 | if not self.haveui and self.lastui and self.timeout and (self.lastui + self.timeout) < time.time(): | ||
176 | print("Server timeout, exiting.") | ||
177 | self.quit = True | ||
141 | 178 | ||
142 | self.event_queue.close() | 179 | ready = self.idle_commands(.1, fds) |
143 | bb.event.unregister_UIHhandler(self.event_handle.value, True) | 180 | |
144 | self.command_channel.close() | 181 | print("Exiting") |
145 | self.cooker.shutdown(True) | 182 | try: |
146 | self.quitout.close() | 183 | self.cooker.shutdown(True) |
184 | except: | ||
185 | pass | ||
186 | |||
187 | # Remove the socket file so we don't get any more connections to avoid races | ||
188 | os.unlink(self.sockname) | ||
189 | self.sock.close() | ||
190 | |||
191 | # Finally release the lockfile but warn about other processes holding it open | ||
192 | lock = self.bitbake_lock | ||
193 | lockfile = lock.name | ||
194 | lock.close() | ||
195 | lock = None | ||
196 | |||
197 | while not lock: | ||
198 | with bb.utils.timeout(3): | ||
199 | lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) | ||
200 | if not lock: | ||
201 | # Some systems may not have lsof available | ||
202 | procs = None | ||
203 | try: | ||
204 | procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) | ||
205 | except OSError as e: | ||
206 | if e.errno != errno.ENOENT: | ||
207 | raise | ||
208 | if procs is None: | ||
209 | # Fall back to fuser if lsof is unavailable | ||
210 | try: | ||
211 | procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) | ||
212 | except OSError as e: | ||
213 | if e.errno != errno.ENOENT: | ||
214 | raise | ||
215 | |||
216 | msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" | ||
217 | if procs: | ||
218 | msg += ":\n%s" % str(procs) | ||
219 | print(msg) | ||
220 | return | ||
221 | # We hold the lock so we can remove the file (hide stale pid data) | ||
222 | bb.utils.remove(lockfile) | ||
223 | bb.utils.unlockfile(lock) | ||
147 | 224 | ||
148 | def idle_commands(self, delay, fds=None): | 225 | def idle_commands(self, delay, fds=None): |
149 | nextsleep = delay | 226 | nextsleep = delay |
@@ -189,140 +266,253 @@ class ProcessServer(Process): | |||
189 | nextsleep = self.next_heartbeat - now | 266 | nextsleep = self.next_heartbeat - now |
190 | 267 | ||
191 | if nextsleep is not None: | 268 | if nextsleep is not None: |
269 | if self.xmlrpc: | ||
270 | nextsleep = self.xmlrpc.get_timeout(nextsleep) | ||
192 | try: | 271 | try: |
193 | select.select(fds,[],[],nextsleep) | 272 | return select.select(fds,[],[],nextsleep)[0] |
194 | except InterruptedError: | 273 | except InterruptedError: |
195 | # ignore EINTR error, nextsleep only used for wait | 274 | # Ignore EINTR |
196 | # certain time | 275 | return [] |
197 | pass | 276 | else: |
277 | return [] | ||
278 | |||
279 | |||
280 | class ServerCommunicator(): | ||
281 | def __init__(self, connection, recv): | ||
282 | self.connection = connection | ||
283 | self.recv = recv | ||
198 | 284 | ||
199 | def runCommand(self, command): | 285 | def runCommand(self, command): |
200 | """ | ||
201 | Run a cooker command on the server | ||
202 | """ | ||
203 | self.command_channel.send(self.cooker.command.runCommand(command)) | ||
204 | 286 | ||
205 | def stop(self): | 287 | self.connection.send(command) |
206 | self.quitin.send("quit") | 288 | while True: |
207 | self.quitin.close() | 289 | # don't let the user ctrl-c while we're waiting for a response |
290 | try: | ||
291 | for idx in range(0,4): # 0, 1, 2, 3 | ||
292 | if self.recv.poll(1): | ||
293 | return self.recv.get() | ||
294 | else: | ||
295 | bb.warn("Timeout while attempting to communicate with bitbake server") | ||
296 | raise ProcessTimeout("Gave up; Too many tries: timeout while attempting to communicate with bitbake server") | ||
297 | except KeyboardInterrupt: | ||
298 | pass | ||
299 | |||
300 | def updateFeatureSet(self, featureset): | ||
301 | _, error = self.runCommand(["setFeatures", featureset]) | ||
302 | if error: | ||
303 | logger.error("Unable to set the cooker to the correct featureset: %s" % error) | ||
304 | raise BaseException(error) | ||
305 | |||
306 | def getEventHandle(self): | ||
307 | handle, error = self.runCommand(["getUIHandlerNum"]) | ||
308 | if error: | ||
309 | logger.error("Unable to get UI Handler Number: %s" % error) | ||
310 | raise BaseException(error) | ||
208 | 311 | ||
209 | def addcooker(self, cooker): | 312 | return handle |
210 | self.cooker = cooker | ||
211 | 313 | ||
212 | def register_idle_function(self, function, data): | 314 | def terminateServer(self): |
213 | """Register a function to be called while the server is idle""" | 315 | self.connection.send(['terminateServer']) |
214 | assert hasattr(function, '__call__') | 316 | return |
215 | self._idlefuns[function] = data | ||
216 | 317 | ||
217 | class BitBakeProcessServerConnection(object): | 318 | class BitBakeProcessServerConnection(object): |
218 | def __init__(self, serverImpl, ui_channel, event_queue): | 319 | def __init__(self, ui_channel, recv, eq): |
219 | self.procserver = serverImpl | 320 | self.connection = ServerCommunicator(ui_channel, recv) |
220 | self.ui_channel = ui_channel | 321 | self.events = eq |
221 | self.event_queue = event_queue | ||
222 | self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver) | ||
223 | self.events = self.event_queue | ||
224 | self.terminated = False | ||
225 | |||
226 | def sigterm_terminate(self): | ||
227 | bb.error("UI received SIGTERM") | ||
228 | self.terminate() | ||
229 | 322 | ||
230 | def terminate(self): | 323 | def terminate(self): |
231 | if self.terminated: | 324 | self.socket_connection.close() |
232 | return | 325 | return |
233 | self.terminated = True | 326 | |
234 | def flushevents(): | 327 | class BitBakeServer(object): |
235 | while True: | 328 | def __init__(self, lock, sockname, configuration, featureset): |
236 | try: | 329 | |
237 | event = self.event_queue.get(block=False) | 330 | self.configuration = configuration |
238 | except (Empty, IOError): | 331 | self.featureset = featureset |
239 | break | 332 | self.sockname = sockname |
240 | if isinstance(event, logging.LogRecord): | 333 | self.bitbake_lock = lock |
241 | logger.handle(event) | 334 | |
242 | 335 | # Create server control socket | |
243 | self.procserver.stop() | 336 | if os.path.exists(sockname): |
244 | 337 | os.unlink(sockname) | |
245 | while self.procserver.is_alive(): | 338 | |
246 | flushevents() | 339 | self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
247 | self.procserver.join(0.1) | 340 | # AF_UNIX has path length issues so chdir here to workaround |
248 | 341 | cwd = os.getcwd() | |
249 | self.ui_channel.close() | ||
250 | self.event_queue.close() | ||
251 | self.event_queue.setexit() | ||
252 | # XXX: Call explicity close in _writer to avoid | ||
253 | # fd leakage because isn't called on Queue.close() | ||
254 | self.event_queue._writer.close() | ||
255 | |||
256 | def setupEventQueue(self): | ||
257 | pass | ||
258 | |||
259 | # Wrap Queue to provide API which isn't server implementation specific | ||
260 | class ProcessEventQueue(multiprocessing.queues.Queue): | ||
261 | def __init__(self, maxsize): | ||
262 | multiprocessing.queues.Queue.__init__(self, maxsize, ctx=multiprocessing.get_context()) | ||
263 | self.exit = False | ||
264 | bb.utils.set_process_name("ProcessEQueue") | ||
265 | |||
266 | def setexit(self): | ||
267 | self.exit = True | ||
268 | |||
269 | def waitEvent(self, timeout): | ||
270 | if self.exit: | ||
271 | return self.getEvent() | ||
272 | try: | 342 | try: |
273 | if not self.server.is_alive(): | 343 | os.chdir(os.path.dirname(sockname)) |
274 | return self.getEvent() | 344 | self.sock.bind(os.path.basename(sockname)) |
275 | if timeout == 0: | 345 | finally: |
276 | return self.get(False) | 346 | os.chdir(cwd) |
277 | return self.get(True, timeout) | 347 | self.sock.listen(1) |
278 | except Empty: | 348 | |
279 | return None | 349 | os.set_inheritable(self.sock.fileno(), True) |
350 | bb.daemonize.createDaemon(self._startServer, "bitbake-cookerdaemon.log") | ||
351 | self.sock.close() | ||
352 | self.bitbake_lock.close() | ||
353 | |||
354 | def _startServer(self): | ||
355 | server = ProcessServer(self.bitbake_lock, self.sock, self.sockname) | ||
356 | self.configuration.setServerRegIdleCallback(server.register_idle_function) | ||
357 | |||
358 | # Copy prefile and postfile to _server variants | ||
359 | for param in ('prefile', 'postfile'): | ||
360 | value = getattr(self.configuration, param) | ||
361 | if value: | ||
362 | setattr(self.configuration, "%s_server" % param, value) | ||
363 | |||
364 | self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset) | ||
365 | server.cooker = self.cooker | ||
366 | server.server_timeout = self.configuration.server_timeout | ||
367 | server.xmlrpcinterface = self.configuration.xmlrpcinterface | ||
368 | print("Started bitbake server pid %d" % os.getpid()) | ||
369 | server.start() | ||
370 | |||
371 | def connectProcessServer(sockname, featureset): | ||
372 | # Connect to socket | ||
373 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | ||
374 | # AF_UNIX has path length issues so chdir here to workaround | ||
375 | cwd = os.getcwd() | ||
376 | |||
377 | try: | ||
378 | os.chdir(os.path.dirname(sockname)) | ||
379 | sock.connect(os.path.basename(sockname)) | ||
380 | finally: | ||
381 | os.chdir(cwd) | ||
382 | |||
383 | try: | ||
384 | # Send an fd for the remote to write events to | ||
385 | readfd, writefd = os.pipe() | ||
386 | eq = BBUIEventQueue(readfd) | ||
387 | # Send an fd for the remote to recieve commands from | ||
388 | readfd1, writefd1 = os.pipe() | ||
389 | command_chan = ConnectionWriter(writefd1) | ||
390 | # Send an fd for the remote to write commands results to | ||
391 | readfd2, writefd2 = os.pipe() | ||
392 | command_chan_recv = ConnectionReader(readfd2) | ||
393 | |||
394 | sendfds(sock, [writefd, readfd1, writefd2]) | ||
395 | |||
396 | server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq) | ||
397 | |||
398 | server_connection.connection.updateFeatureSet(featureset) | ||
399 | |||
400 | # Save sock so it doesn't get gc'd for the life of our connection | ||
401 | server_connection.socket_connection = sock | ||
402 | except: | ||
403 | sock.close() | ||
404 | raise | ||
405 | |||
406 | return server_connection | ||
407 | |||
408 | def sendfds(sock, fds): | ||
409 | '''Send an array of fds over an AF_UNIX socket.''' | ||
410 | fds = array.array('i', fds) | ||
411 | msg = bytes([len(fds) % 256]) | ||
412 | sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) | ||
413 | |||
414 | def recvfds(sock, size): | ||
415 | '''Receive an array of fds over an AF_UNIX socket.''' | ||
416 | a = array.array('i') | ||
417 | bytes_size = a.itemsize * size | ||
418 | msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size)) | ||
419 | if not msg and not ancdata: | ||
420 | raise EOFError | ||
421 | try: | ||
422 | if len(ancdata) != 1: | ||
423 | raise RuntimeError('received %d items of ancdata' % | ||
424 | len(ancdata)) | ||
425 | cmsg_level, cmsg_type, cmsg_data = ancdata[0] | ||
426 | if (cmsg_level == socket.SOL_SOCKET and | ||
427 | cmsg_type == socket.SCM_RIGHTS): | ||
428 | if len(cmsg_data) % a.itemsize != 0: | ||
429 | raise ValueError | ||
430 | a.frombytes(cmsg_data) | ||
431 | assert len(a) % 256 == msg[0] | ||
432 | return list(a) | ||
433 | except (ValueError, IndexError): | ||
434 | pass | ||
435 | raise RuntimeError('Invalid data received') | ||
436 | |||
437 | class BBUIEventQueue: | ||
438 | def __init__(self, readfd): | ||
439 | |||
440 | self.eventQueue = [] | ||
441 | self.eventQueueLock = threading.Lock() | ||
442 | self.eventQueueNotify = threading.Event() | ||
443 | |||
444 | self.reader = ConnectionReader(readfd) | ||
445 | |||
446 | self.t = threading.Thread() | ||
447 | self.t.setDaemon(True) | ||
448 | self.t.run = self.startCallbackHandler | ||
449 | self.t.start() | ||
280 | 450 | ||
281 | def getEvent(self): | 451 | def getEvent(self): |
282 | try: | 452 | self.eventQueueLock.acquire() |
283 | if not self.server.is_alive(): | 453 | |
284 | self.setexit() | 454 | if len(self.eventQueue) == 0: |
285 | return self.get(False) | 455 | self.eventQueueLock.release() |
286 | except Empty: | ||
287 | if self.exit: | ||
288 | sys.exit(1) | ||
289 | return None | 456 | return None |
290 | 457 | ||
291 | class BitBakeServer(object): | 458 | item = self.eventQueue.pop(0) |
292 | def initServer(self, single_use=True): | ||
293 | # establish communication channels. We use bidirectional pipes for | ||
294 | # ui <--> server command/response pairs | ||
295 | # and a queue for server -> ui event notifications | ||
296 | # | ||
297 | self.ui_channel, self.server_channel = Pipe() | ||
298 | self.event_queue = ProcessEventQueue(0) | ||
299 | self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None) | ||
300 | self.event_queue.server = self.serverImpl | ||
301 | |||
302 | def detach(self): | ||
303 | self.serverImpl.start() | ||
304 | return | ||
305 | 459 | ||
306 | def establishConnection(self, featureset): | 460 | if len(self.eventQueue) == 0: |
461 | self.eventQueueNotify.clear() | ||
307 | 462 | ||
308 | self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue) | 463 | self.eventQueueLock.release() |
464 | return item | ||
309 | 465 | ||
310 | _, error = self.connection.connection.runCommand(["setFeatures", featureset]) | 466 | def waitEvent(self, delay): |
311 | if error: | 467 | self.eventQueueNotify.wait(delay) |
312 | logger.error("Unable to set the cooker to the correct featureset: %s" % error) | 468 | return self.getEvent() |
313 | raise BaseException(error) | ||
314 | signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate()) | ||
315 | return self.connection | ||
316 | 469 | ||
317 | def addcooker(self, cooker): | 470 | def queue_event(self, event): |
318 | self.cooker = cooker | 471 | self.eventQueueLock.acquire() |
319 | self.serverImpl.addcooker(cooker) | 472 | self.eventQueue.append(event) |
473 | self.eventQueueNotify.set() | ||
474 | self.eventQueueLock.release() | ||
320 | 475 | ||
321 | def getServerIdleCB(self): | 476 | def send_event(self, event): |
322 | return self.serverImpl.register_idle_function | 477 | self.queue_event(pickle.loads(event)) |
323 | 478 | ||
324 | def saveConnectionDetails(self): | 479 | def startCallbackHandler(self): |
325 | return | 480 | bb.utils.set_process_name("UIEventQueue") |
481 | while True: | ||
482 | self.reader.wait() | ||
483 | event = self.reader.get() | ||
484 | self.queue_event(event) | ||
485 | |||
486 | class ConnectionReader(object): | ||
487 | |||
488 | def __init__(self, fd): | ||
489 | self.reader = multiprocessing.connection.Connection(fd, writable=False) | ||
490 | self.rlock = multiprocessing.Lock() | ||
491 | |||
492 | def wait(self, timeout=None): | ||
493 | return multiprocessing.connection.wait([self.reader], timeout) | ||
494 | |||
495 | def poll(self, timeout=None): | ||
496 | return self.reader.poll(timeout) | ||
497 | |||
498 | def get(self): | ||
499 | with self.rlock: | ||
500 | res = self.reader.recv_bytes() | ||
501 | return multiprocessing.reduction.ForkingPickler.loads(res) | ||
502 | |||
503 | def fileno(self): | ||
504 | return self.reader.fileno() | ||
505 | |||
506 | class ConnectionWriter(object): | ||
507 | |||
508 | def __init__(self, fd): | ||
509 | self.writer = multiprocessing.connection.Connection(fd, readable=False) | ||
510 | self.wlock = multiprocessing.Lock() | ||
511 | # Why bb.event needs this I have no idea | ||
512 | self.event = self | ||
513 | |||
514 | def send(self, obj): | ||
515 | obj = multiprocessing.reduction.ForkingPickler.dumps(obj) | ||
516 | with self.wlock: | ||
517 | self.writer.send_bytes(obj) | ||
326 | 518 | ||
327 | def endSession(self): | ||
328 | self.connection.terminate() | ||
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py deleted file mode 100644 index 6874765136..0000000000 --- a/bitbake/lib/bb/server/xmlrpc.py +++ /dev/null | |||
@@ -1,492 +0,0 @@ | |||
1 | # | ||
2 | # BitBake XMLRPC Server | ||
3 | # | ||
4 | # Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer | ||
5 | # Copyright (C) 2006 - 2008 Richard Purdie | ||
6 | # | ||
7 | # This program is free software; you can redistribute it and/or modify | ||
8 | # it under the terms of the GNU General Public License version 2 as | ||
9 | # published by the Free Software Foundation. | ||
10 | # | ||
11 | # This program is distributed in the hope that it will be useful, | ||
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | # GNU General Public License for more details. | ||
15 | # | ||
16 | # You should have received a copy of the GNU General Public License along | ||
17 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
19 | |||
20 | """ | ||
21 | This module implements an xmlrpc server for BitBake. | ||
22 | |||
23 | Use this by deriving a class from BitBakeXMLRPCServer and then adding | ||
24 | methods which you want to "export" via XMLRPC. If the methods have the | ||
25 | prefix xmlrpc_, then registering those function will happen automatically, | ||
26 | if not, you need to call register_function. | ||
27 | |||
28 | Use register_idle_function() to add a function which the xmlrpc server | ||
29 | calls from within server_forever when no requests are pending. Make sure | ||
30 | that those functions are non-blocking or else you will introduce latency | ||
31 | in the server's main loop. | ||
32 | """ | ||
33 | |||
34 | import os | ||
35 | import sys | ||
36 | |||
37 | import hashlib | ||
38 | import time | ||
39 | import socket | ||
40 | import signal | ||
41 | import threading | ||
42 | import pickle | ||
43 | import inspect | ||
44 | import select | ||
45 | import http.client | ||
46 | import xmlrpc.client | ||
47 | from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler | ||
48 | |||
49 | import bb | ||
50 | from bb import daemonize | ||
51 | from bb.ui import uievent | ||
52 | |||
53 | DEBUG = False | ||
54 | |||
55 | class BBTransport(xmlrpc.client.Transport): | ||
56 | def __init__(self, timeout): | ||
57 | self.timeout = timeout | ||
58 | self.connection_token = None | ||
59 | xmlrpc.client.Transport.__init__(self) | ||
60 | |||
61 | # Modified from default to pass timeout to HTTPConnection | ||
62 | def make_connection(self, host): | ||
63 | #return an existing connection if possible. This allows | ||
64 | #HTTP/1.1 keep-alive. | ||
65 | if self._connection and host == self._connection[0]: | ||
66 | return self._connection[1] | ||
67 | |||
68 | # create a HTTP connection object from a host descriptor | ||
69 | chost, self._extra_headers, x509 = self.get_host_info(host) | ||
70 | #store the host argument along with the connection object | ||
71 | self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout) | ||
72 | return self._connection[1] | ||
73 | |||
74 | def set_connection_token(self, token): | ||
75 | self.connection_token = token | ||
76 | |||
77 | def send_content(self, h, body): | ||
78 | if self.connection_token: | ||
79 | h.putheader("Bitbake-token", self.connection_token) | ||
80 | xmlrpc.client.Transport.send_content(self, h, body) | ||
81 | |||
82 | def _create_server(host, port, timeout = 60): | ||
83 | t = BBTransport(timeout) | ||
84 | s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True) | ||
85 | return s, t | ||
86 | |||
87 | def check_connection(remote, timeout): | ||
88 | try: | ||
89 | host, port = remote.split(":") | ||
90 | port = int(port) | ||
91 | except Exception as e: | ||
92 | bb.warn("Failed to read remote definition (%s)" % str(e)) | ||
93 | raise e | ||
94 | |||
95 | server, _transport = _create_server(host, port, timeout) | ||
96 | try: | ||
97 | ret, err = server.runCommand(['getVariable', 'TOPDIR']) | ||
98 | if err or not ret: | ||
99 | return False | ||
100 | except ConnectionError: | ||
101 | return False | ||
102 | return True | ||
103 | |||
104 | class BitBakeServerCommands(): | ||
105 | |||
106 | def __init__(self, server): | ||
107 | self.server = server | ||
108 | self.has_client = False | ||
109 | |||
110 | def registerEventHandler(self, host, port): | ||
111 | """ | ||
112 | Register a remote UI Event Handler | ||
113 | """ | ||
114 | s, t = _create_server(host, port) | ||
115 | |||
116 | # we don't allow connections if the cooker is running | ||
117 | if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]): | ||
118 | return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state) | ||
119 | |||
120 | self.event_handle = bb.event.register_UIHhandler(s, True) | ||
121 | return self.event_handle, 'OK' | ||
122 | |||
123 | def unregisterEventHandler(self, handlerNum): | ||
124 | """ | ||
125 | Unregister a remote UI Event Handler | ||
126 | """ | ||
127 | return bb.event.unregister_UIHhandler(handlerNum, True) | ||
128 | |||
129 | def runCommand(self, command): | ||
130 | """ | ||
131 | Run a cooker command on the server | ||
132 | """ | ||
133 | return self.cooker.command.runCommand(command, self.server.readonly) | ||
134 | |||
135 | def getEventHandle(self): | ||
136 | return self.event_handle | ||
137 | |||
138 | def terminateServer(self): | ||
139 | """ | ||
140 | Trigger the server to quit | ||
141 | """ | ||
142 | self.server.quit = True | ||
143 | print("Server (cooker) exiting") | ||
144 | return | ||
145 | |||
146 | def addClient(self): | ||
147 | if self.has_client: | ||
148 | return None | ||
149 | token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest() | ||
150 | self.server.set_connection_token(token) | ||
151 | self.has_client = True | ||
152 | return token | ||
153 | |||
154 | def removeClient(self): | ||
155 | if self.has_client: | ||
156 | self.server.set_connection_token(None) | ||
157 | self.has_client = False | ||
158 | if self.server.single_use: | ||
159 | self.server.quit = True | ||
160 | |||
161 | # This request handler checks if the request has a "Bitbake-token" header | ||
162 | # field (this comes from the client side) and compares it with its internal | ||
163 | # "Bitbake-token" field (this comes from the server). If the two are not | ||
164 | # equal, it is assumed that a client is trying to connect to the server | ||
165 | # while another client is connected to the server. In this case, a 503 error | ||
166 | # ("service unavailable") is returned to the client. | ||
167 | class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): | ||
168 | def __init__(self, request, client_address, server): | ||
169 | self.server = server | ||
170 | SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) | ||
171 | |||
172 | def do_POST(self): | ||
173 | try: | ||
174 | remote_token = self.headers["Bitbake-token"] | ||
175 | except: | ||
176 | remote_token = None | ||
177 | if remote_token != self.server.connection_token and remote_token != "observer": | ||
178 | self.report_503() | ||
179 | else: | ||
180 | if remote_token == "observer": | ||
181 | self.server.readonly = True | ||
182 | else: | ||
183 | self.server.readonly = False | ||
184 | SimpleXMLRPCRequestHandler.do_POST(self) | ||
185 | |||
186 | def report_503(self): | ||
187 | self.send_response(503) | ||
188 | response = 'No more client allowed' | ||
189 | self.send_header("Content-type", "text/plain") | ||
190 | self.send_header("Content-length", str(len(response))) | ||
191 | self.end_headers() | ||
192 | self.wfile.write(bytes(response, 'utf-8')) | ||
193 | |||
194 | |||
195 | class XMLRPCProxyServer(object): | ||
196 | """ not a real working server, but a stub for a proxy server connection | ||
197 | |||
198 | """ | ||
199 | def __init__(self, host, port, use_builtin_types=True): | ||
200 | self.host = host | ||
201 | self.port = port | ||
202 | self._idlefuns = {} | ||
203 | |||
204 | def addcooker(self, cooker): | ||
205 | self.cooker = cooker | ||
206 | |||
207 | def register_idle_function(self, function, data): | ||
208 | """Register a function to be called while the server is idle""" | ||
209 | assert hasattr(function, '__call__') | ||
210 | self._idlefuns[function] = data | ||
211 | |||
212 | |||
213 | class XMLRPCServer(SimpleXMLRPCServer): | ||
214 | # remove this when you're done with debugging | ||
215 | # allow_reuse_address = True | ||
216 | |||
217 | def __init__(self, interface, single_use=False, idle_timeout=0): | ||
218 | """ | ||
219 | Constructor | ||
220 | """ | ||
221 | self._idlefuns = {} | ||
222 | self.single_use = single_use | ||
223 | # Use auto port configuration | ||
224 | if (interface[1] == -1): | ||
225 | interface = (interface[0], 0) | ||
226 | SimpleXMLRPCServer.__init__(self, interface, | ||
227 | requestHandler=BitBakeXMLRPCRequestHandler, | ||
228 | logRequests=False, allow_none=True) | ||
229 | self.host, self.port = self.socket.getsockname() | ||
230 | self.connection_token = None | ||
231 | #self.register_introspection_functions() | ||
232 | self.commands = BitBakeServerCommands(self) | ||
233 | self.autoregister_all_functions(self.commands, "") | ||
234 | self.interface = interface | ||
235 | self.time = time.time() | ||
236 | self.idle_timeout = idle_timeout | ||
237 | if idle_timeout: | ||
238 | self.register_idle_function(self.handle_idle_timeout, self) | ||
239 | self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore. | ||
240 | self.next_heartbeat = time.time() | ||
241 | |||
242 | def addcooker(self, cooker): | ||
243 | self.cooker = cooker | ||
244 | self.commands.cooker = cooker | ||
245 | |||
246 | def autoregister_all_functions(self, context, prefix): | ||
247 | """ | ||
248 | Convenience method for registering all functions in the scope | ||
249 | of this class that start with a common prefix | ||
250 | """ | ||
251 | methodlist = inspect.getmembers(context, inspect.ismethod) | ||
252 | for name, method in methodlist: | ||
253 | if name.startswith(prefix): | ||
254 | self.register_function(method, name[len(prefix):]) | ||
255 | |||
256 | def handle_idle_timeout(self, server, data, abort): | ||
257 | if not abort: | ||
258 | if time.time() - server.time > server.idle_timeout: | ||
259 | server.quit = True | ||
260 | print("Server idle timeout expired") | ||
261 | return [] | ||
262 | |||
263 | def serve_forever(self): | ||
264 | heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') | ||
265 | if heartbeat_event: | ||
266 | try: | ||
267 | self.heartbeat_seconds = float(heartbeat_event) | ||
268 | except: | ||
269 | # Throwing an exception here causes bitbake to hang. | ||
270 | # Just warn about the invalid setting and continue | ||
271 | bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) | ||
272 | |||
273 | # Start the actual XMLRPC server | ||
274 | bb.cooker.server_main(self.cooker, self._serve_forever) | ||
275 | |||
276 | def _serve_forever(self): | ||
277 | """ | ||
278 | Serve Requests. Overloaded to honor a quit command | ||
279 | """ | ||
280 | self.quit = False | ||
281 | while not self.quit: | ||
282 | fds = [self] | ||
283 | nextsleep = 0.1 | ||
284 | for function, data in list(self._idlefuns.items()): | ||
285 | retval = None | ||
286 | try: | ||
287 | retval = function(self, data, False) | ||
288 | if retval is False: | ||
289 | del self._idlefuns[function] | ||
290 | elif retval is True: | ||
291 | nextsleep = 0 | ||
292 | elif isinstance(retval, float): | ||
293 | if (retval < nextsleep): | ||
294 | nextsleep = retval | ||
295 | else: | ||
296 | fds = fds + retval | ||
297 | except SystemExit: | ||
298 | raise | ||
299 | except: | ||
300 | import traceback | ||
301 | traceback.print_exc() | ||
302 | if retval == None: | ||
303 | # the function execute failed; delete it | ||
304 | del self._idlefuns[function] | ||
305 | pass | ||
306 | |||
307 | socktimeout = self.socket.gettimeout() or nextsleep | ||
308 | socktimeout = min(socktimeout, nextsleep) | ||
309 | # Mirror what BaseServer handle_request would do | ||
310 | try: | ||
311 | fd_sets = select.select(fds, [], [], socktimeout) | ||
312 | if fd_sets[0] and self in fd_sets[0]: | ||
313 | if self.idle_timeout: | ||
314 | self.time = time.time() | ||
315 | self._handle_request_noblock() | ||
316 | except IOError: | ||
317 | # we ignore interrupted calls | ||
318 | pass | ||
319 | |||
320 | # Create new heartbeat event? | ||
321 | now = time.time() | ||
322 | if now >= self.next_heartbeat: | ||
323 | # We might have missed heartbeats. Just trigger once in | ||
324 | # that case and continue after the usual delay. | ||
325 | self.next_heartbeat += self.heartbeat_seconds | ||
326 | if self.next_heartbeat <= now: | ||
327 | self.next_heartbeat = now + self.heartbeat_seconds | ||
328 | heartbeat = bb.event.HeartbeatEvent(now) | ||
329 | bb.event.fire(heartbeat, self.cooker.data) | ||
330 | if nextsleep and now + nextsleep > self.next_heartbeat: | ||
331 | # Shorten timeout so that we we wake up in time for | ||
332 | # the heartbeat. | ||
333 | nextsleep = self.next_heartbeat - now | ||
334 | |||
335 | # Tell idle functions we're exiting | ||
336 | for function, data in list(self._idlefuns.items()): | ||
337 | try: | ||
338 | retval = function(self, data, True) | ||
339 | except: | ||
340 | pass | ||
341 | self.server_close() | ||
342 | return | ||
343 | |||
344 | def set_connection_token(self, token): | ||
345 | self.connection_token = token | ||
346 | |||
347 | def register_idle_function(self, function, data): | ||
348 | """Register a function to be called while the server is idle""" | ||
349 | assert hasattr(function, '__call__') | ||
350 | self._idlefuns[function] = data | ||
351 | |||
352 | |||
353 | class BitBakeXMLRPCServerConnection(object): | ||
354 | def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None): | ||
355 | self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port) | ||
356 | self.clientinfo = clientinfo | ||
357 | self.serverImpl = serverImpl | ||
358 | self.observer_only = observer_only | ||
359 | if featureset: | ||
360 | self.featureset = featureset | ||
361 | else: | ||
362 | self.featureset = [] | ||
363 | |||
364 | def connect(self, token = None): | ||
365 | if token is None: | ||
366 | if self.observer_only: | ||
367 | token = "observer" | ||
368 | else: | ||
369 | token = self.connection.addClient() | ||
370 | |||
371 | if token is None: | ||
372 | return None | ||
373 | |||
374 | self.transport.set_connection_token(token) | ||
375 | return self | ||
376 | |||
377 | def setupEventQueue(self): | ||
378 | self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo) | ||
379 | for event in bb.event.ui_queue: | ||
380 | self.events.queue_event(event) | ||
381 | |||
382 | _, error = self.connection.runCommand(["setFeatures", self.featureset]) | ||
383 | if error: | ||
384 | # disconnect the client, we can't make the setFeature work | ||
385 | self.connection.removeClient() | ||
386 | # no need to log it here, the error shall be sent to the client | ||
387 | raise BaseException(error) | ||
388 | |||
389 | def removeClient(self): | ||
390 | if not self.observer_only: | ||
391 | self.connection.removeClient() | ||
392 | |||
393 | def terminate(self): | ||
394 | # Don't wait for server indefinitely | ||
395 | import socket | ||
396 | socket.setdefaulttimeout(2) | ||
397 | try: | ||
398 | self.events.system_quit() | ||
399 | except: | ||
400 | pass | ||
401 | try: | ||
402 | self.connection.removeClient() | ||
403 | except: | ||
404 | pass | ||
405 | |||
406 | class BitBakeServer(object): | ||
407 | def initServer(self, interface = ("localhost", 0), | ||
408 | single_use = False, idle_timeout=0): | ||
409 | self.interface = interface | ||
410 | self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout) | ||
411 | |||
412 | def detach(self): | ||
413 | daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log") | ||
414 | del self.cooker | ||
415 | |||
416 | def establishConnection(self, featureset): | ||
417 | self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset) | ||
418 | return self.connection.connect() | ||
419 | |||
420 | def set_connection_token(self, token): | ||
421 | self.connection.transport.set_connection_token(token) | ||
422 | |||
423 | def addcooker(self, cooker): | ||
424 | self.cooker = cooker | ||
425 | self.serverImpl.addcooker(cooker) | ||
426 | |||
427 | def getServerIdleCB(self): | ||
428 | return self.serverImpl.register_idle_function | ||
429 | |||
430 | def saveConnectionDetails(self): | ||
431 | return | ||
432 | |||
433 | def endSession(self): | ||
434 | self.connection.terminate() | ||
435 | |||
436 | class BitBakeXMLRPCClient(object): | ||
437 | |||
438 | def __init__(self, observer_only = False, token = None): | ||
439 | self.token = token | ||
440 | |||
441 | self.observer_only = observer_only | ||
442 | # if we need extra caches, just tell the server to load them all | ||
443 | pass | ||
444 | |||
445 | def saveConnectionDetails(self, remote): | ||
446 | self.remote = remote | ||
447 | |||
448 | def establishConnection(self, featureset): | ||
449 | # The format of "remote" must be "server:port" | ||
450 | try: | ||
451 | [host, port] = self.remote.split(":") | ||
452 | port = int(port) | ||
453 | except Exception as e: | ||
454 | bb.warn("Failed to read remote definition (%s)" % str(e)) | ||
455 | raise e | ||
456 | |||
457 | # We need our IP for the server connection. We get the IP | ||
458 | # by trying to connect with the server | ||
459 | try: | ||
460 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) | ||
461 | s.connect((host, port)) | ||
462 | ip = s.getsockname()[0] | ||
463 | s.close() | ||
464 | except Exception as e: | ||
465 | bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e))) | ||
466 | raise e | ||
467 | try: | ||
468 | self.serverImpl = XMLRPCProxyServer(host, port, use_builtin_types=True) | ||
469 | self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset) | ||
470 | return self.connection.connect(self.token) | ||
471 | except Exception as e: | ||
472 | bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e))) | ||
473 | raise e | ||
474 | |||
475 | def endSession(self): | ||
476 | self.connection.removeClient() | ||
477 | |||
478 | def initServer(self): | ||
479 | self.serverImpl = None | ||
480 | self.connection = None | ||
481 | return | ||
482 | |||
483 | def addcooker(self, cooker): | ||
484 | self.cooker = cooker | ||
485 | self.serverImpl.addcooker(cooker) | ||
486 | |||
487 | def getServerIdleCB(self): | ||
488 | return self.serverImpl.register_idle_function | ||
489 | |||
490 | def detach(self): | ||
491 | return | ||
492 | |||
diff --git a/bitbake/lib/bb/server/xmlrpcclient.py b/bitbake/lib/bb/server/xmlrpcclient.py new file mode 100644 index 0000000000..4661a9e5a0 --- /dev/null +++ b/bitbake/lib/bb/server/xmlrpcclient.py | |||
@@ -0,0 +1,154 @@ | |||
1 | # | ||
2 | # BitBake XMLRPC Client Interface | ||
3 | # | ||
4 | # Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer | ||
5 | # Copyright (C) 2006 - 2008 Richard Purdie | ||
6 | # | ||
7 | # This program is free software; you can redistribute it and/or modify | ||
8 | # it under the terms of the GNU General Public License version 2 as | ||
9 | # published by the Free Software Foundation. | ||
10 | # | ||
11 | # This program is distributed in the hope that it will be useful, | ||
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | # GNU General Public License for more details. | ||
15 | # | ||
16 | # You should have received a copy of the GNU General Public License along | ||
17 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
19 | |||
20 | import os | ||
21 | import sys | ||
22 | |||
23 | import socket | ||
24 | import http.client | ||
25 | import xmlrpc.client | ||
26 | |||
27 | import bb | ||
28 | from bb.ui import uievent | ||
29 | |||
30 | class BBTransport(xmlrpc.client.Transport): | ||
31 | def __init__(self, timeout): | ||
32 | self.timeout = timeout | ||
33 | self.connection_token = None | ||
34 | xmlrpc.client.Transport.__init__(self) | ||
35 | |||
36 | # Modified from default to pass timeout to HTTPConnection | ||
37 | def make_connection(self, host): | ||
38 | #return an existing connection if possible. This allows | ||
39 | #HTTP/1.1 keep-alive. | ||
40 | if self._connection and host == self._connection[0]: | ||
41 | return self._connection[1] | ||
42 | |||
43 | # create a HTTP connection object from a host descriptor | ||
44 | chost, self._extra_headers, x509 = self.get_host_info(host) | ||
45 | #store the host argument along with the connection object | ||
46 | self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout) | ||
47 | return self._connection[1] | ||
48 | |||
49 | def set_connection_token(self, token): | ||
50 | self.connection_token = token | ||
51 | |||
52 | def send_content(self, h, body): | ||
53 | if self.connection_token: | ||
54 | h.putheader("Bitbake-token", self.connection_token) | ||
55 | xmlrpc.client.Transport.send_content(self, h, body) | ||
56 | |||
57 | def _create_server(host, port, timeout = 60): | ||
58 | t = BBTransport(timeout) | ||
59 | s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True) | ||
60 | return s, t | ||
61 | |||
62 | def check_connection(remote, timeout): | ||
63 | try: | ||
64 | host, port = remote.split(":") | ||
65 | port = int(port) | ||
66 | except Exception as e: | ||
67 | bb.warn("Failed to read remote definition (%s)" % str(e)) | ||
68 | raise e | ||
69 | |||
70 | server, _transport = _create_server(host, port, timeout) | ||
71 | try: | ||
72 | ret, err = server.runCommand(['getVariable', 'TOPDIR']) | ||
73 | if err or not ret: | ||
74 | return False | ||
75 | except ConnectionError: | ||
76 | return False | ||
77 | return True | ||
78 | |||
79 | class BitBakeXMLRPCServerConnection(object): | ||
80 | def __init__(self, host, port, clientinfo=("localhost", 0), observer_only = False, featureset = None): | ||
81 | self.connection, self.transport = _create_server(host, port) | ||
82 | self.clientinfo = clientinfo | ||
83 | self.observer_only = observer_only | ||
84 | if featureset: | ||
85 | self.featureset = featureset | ||
86 | else: | ||
87 | self.featureset = [] | ||
88 | |||
89 | self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo) | ||
90 | |||
91 | _, error = self.connection.runCommand(["setFeatures", self.featureset]) | ||
92 | if error: | ||
93 | # disconnect the client, we can't make the setFeature work | ||
94 | self.connection.removeClient() | ||
95 | # no need to log it here, the error shall be sent to the client | ||
96 | raise BaseException(error) | ||
97 | |||
98 | def connect(self, token = None): | ||
99 | if token is None: | ||
100 | if self.observer_only: | ||
101 | token = "observer" | ||
102 | else: | ||
103 | token = self.connection.addClient() | ||
104 | |||
105 | if token is None: | ||
106 | return None | ||
107 | |||
108 | self.transport.set_connection_token(token) | ||
109 | return self | ||
110 | |||
111 | def removeClient(self): | ||
112 | if not self.observer_only: | ||
113 | self.connection.removeClient() | ||
114 | |||
115 | def terminate(self): | ||
116 | # Don't wait for server indefinitely | ||
117 | socket.setdefaulttimeout(2) | ||
118 | try: | ||
119 | self.events.system_quit() | ||
120 | except: | ||
121 | pass | ||
122 | try: | ||
123 | self.connection.removeClient() | ||
124 | except: | ||
125 | pass | ||
126 | |||
127 | def connectXMLRPC(remote, featureset, observer_only = False, token = None): | ||
128 | # The format of "remote" must be "server:port" | ||
129 | try: | ||
130 | [host, port] = remote.split(":") | ||
131 | port = int(port) | ||
132 | except Exception as e: | ||
133 | bb.warn("Failed to parse remote definition %s (%s)" % (remote, str(e))) | ||
134 | raise e | ||
135 | |||
136 | # We need our IP for the server connection. We get the IP | ||
137 | # by trying to connect with the server | ||
138 | try: | ||
139 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) | ||
140 | s.connect((host, port)) | ||
141 | ip = s.getsockname()[0] | ||
142 | s.close() | ||
143 | except Exception as e: | ||
144 | bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e))) | ||
145 | raise e | ||
146 | try: | ||
147 | connection = BitBakeXMLRPCServerConnection(host, port, (ip, 0), observer_only, featureset) | ||
148 | return connection.connect(token) | ||
149 | except Exception as e: | ||
150 | bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e))) | ||
151 | raise e | ||
152 | |||
153 | |||
154 | |||
diff --git a/bitbake/lib/bb/server/xmlrpcserver.py b/bitbake/lib/bb/server/xmlrpcserver.py new file mode 100644 index 0000000000..875b1282e5 --- /dev/null +++ b/bitbake/lib/bb/server/xmlrpcserver.py | |||
@@ -0,0 +1,158 @@ | |||
1 | # | ||
2 | # BitBake XMLRPC Server Interface | ||
3 | # | ||
4 | # Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer | ||
5 | # Copyright (C) 2006 - 2008 Richard Purdie | ||
6 | # | ||
7 | # This program is free software; you can redistribute it and/or modify | ||
8 | # it under the terms of the GNU General Public License version 2 as | ||
9 | # published by the Free Software Foundation. | ||
10 | # | ||
11 | # This program is distributed in the hope that it will be useful, | ||
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | # GNU General Public License for more details. | ||
15 | # | ||
16 | # You should have received a copy of the GNU General Public License along | ||
17 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
19 | |||
20 | import os | ||
21 | import sys | ||
22 | |||
23 | import hashlib | ||
24 | import time | ||
25 | import inspect | ||
26 | from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler | ||
27 | |||
28 | import bb | ||
29 | |||
30 | # This request handler checks if the request has a "Bitbake-token" header | ||
31 | # field (this comes from the client side) and compares it with its internal | ||
32 | # "Bitbake-token" field (this comes from the server). If the two are not | ||
33 | # equal, it is assumed that a client is trying to connect to the server | ||
34 | # while another client is connected to the server. In this case, a 503 error | ||
35 | # ("service unavailable") is returned to the client. | ||
36 | class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): | ||
37 | def __init__(self, request, client_address, server): | ||
38 | self.server = server | ||
39 | SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) | ||
40 | |||
41 | def do_POST(self): | ||
42 | try: | ||
43 | remote_token = self.headers["Bitbake-token"] | ||
44 | except: | ||
45 | remote_token = None | ||
46 | if 0 and remote_token != self.server.connection_token and remote_token != "observer": | ||
47 | self.report_503() | ||
48 | else: | ||
49 | if remote_token == "observer": | ||
50 | self.server.readonly = True | ||
51 | else: | ||
52 | self.server.readonly = False | ||
53 | SimpleXMLRPCRequestHandler.do_POST(self) | ||
54 | |||
55 | def report_503(self): | ||
56 | self.send_response(503) | ||
57 | response = 'No more client allowed' | ||
58 | self.send_header("Content-type", "text/plain") | ||
59 | self.send_header("Content-length", str(len(response))) | ||
60 | self.end_headers() | ||
61 | self.wfile.write(bytes(response, 'utf-8')) | ||
62 | |||
63 | class BitBakeXMLRPCServer(SimpleXMLRPCServer): | ||
64 | # remove this when you're done with debugging | ||
65 | # allow_reuse_address = True | ||
66 | |||
67 | def __init__(self, interface, cooker, parent): | ||
68 | # Use auto port configuration | ||
69 | if (interface[1] == -1): | ||
70 | interface = (interface[0], 0) | ||
71 | SimpleXMLRPCServer.__init__(self, interface, | ||
72 | requestHandler=BitBakeXMLRPCRequestHandler, | ||
73 | logRequests=False, allow_none=True) | ||
74 | self.host, self.port = self.socket.getsockname() | ||
75 | self.interface = interface | ||
76 | |||
77 | self.connection_token = None | ||
78 | self.commands = BitBakeXMLRPCServerCommands(self) | ||
79 | self.register_functions(self.commands, "") | ||
80 | |||
81 | self.cooker = cooker | ||
82 | self.parent = parent | ||
83 | |||
84 | |||
85 | def register_functions(self, context, prefix): | ||
86 | """ | ||
87 | Convenience method for registering all functions in the scope | ||
88 | of this class that start with a common prefix | ||
89 | """ | ||
90 | methodlist = inspect.getmembers(context, inspect.ismethod) | ||
91 | for name, method in methodlist: | ||
92 | if name.startswith(prefix): | ||
93 | self.register_function(method, name[len(prefix):]) | ||
94 | |||
95 | def get_timeout(self, delay): | ||
96 | socktimeout = self.socket.gettimeout() or delay | ||
97 | return min(socktimeout, delay) | ||
98 | |||
99 | def handle_requests(self): | ||
100 | self._handle_request_noblock() | ||
101 | |||
102 | class BitBakeXMLRPCServerCommands(): | ||
103 | |||
104 | def __init__(self, server): | ||
105 | self.server = server | ||
106 | self.has_client = False | ||
107 | |||
108 | def registerEventHandler(self, host, port): | ||
109 | """ | ||
110 | Register a remote UI Event Handler | ||
111 | """ | ||
112 | s, t = bb.server.xmlrpcclient._create_server(host, port) | ||
113 | |||
114 | # we don't allow connections if the cooker is running | ||
115 | if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]): | ||
116 | return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state) | ||
117 | |||
118 | self.event_handle = bb.event.register_UIHhandler(s, True) | ||
119 | return self.event_handle, 'OK' | ||
120 | |||
121 | def unregisterEventHandler(self, handlerNum): | ||
122 | """ | ||
123 | Unregister a remote UI Event Handler | ||
124 | """ | ||
125 | ret = bb.event.unregister_UIHhandler(handlerNum, True) | ||
126 | self.event_handle = None | ||
127 | return ret | ||
128 | |||
129 | def runCommand(self, command): | ||
130 | """ | ||
131 | Run a cooker command on the server | ||
132 | """ | ||
133 | return self.server.cooker.command.runCommand(command, self.server.readonly) | ||
134 | |||
135 | def getEventHandle(self): | ||
136 | return self.event_handle | ||
137 | |||
138 | def terminateServer(self): | ||
139 | """ | ||
140 | Trigger the server to quit | ||
141 | """ | ||
142 | self.server.parent.quit = True | ||
143 | print("XMLRPC Server triggering exit") | ||
144 | return | ||
145 | |||
146 | def addClient(self): | ||
147 | if self.server.parent.haveui: | ||
148 | return None | ||
149 | token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest() | ||
150 | self.server.connection_token = token | ||
151 | self.server.parent.haveui = True | ||
152 | return token | ||
153 | |||
154 | def removeClient(self): | ||
155 | if self.server.parent.haveui: | ||
156 | self.server.connection_token = None | ||
157 | self.server.parent.haveui = False | ||
158 | |||
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index fb0da62243..303ce02b00 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
@@ -243,7 +243,7 @@ class Tinfoil: | |||
243 | cookerconfig = CookerConfiguration() | 243 | cookerconfig = CookerConfiguration() |
244 | cookerconfig.setConfigParameters(config_params) | 244 | cookerconfig.setConfigParameters(config_params) |
245 | 245 | ||
246 | server, self.server_connection, ui_module = setup_bitbake(config_params, | 246 | self.server_connection, ui_module = setup_bitbake(config_params, |
247 | cookerconfig, | 247 | cookerconfig, |
248 | extrafeatures, | 248 | extrafeatures, |
249 | setup_logging=False) | 249 | setup_logging=False) |
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py index a7efa58bc7..5f061c2623 100644 --- a/bitbake/lib/prserv/serv.py +++ b/bitbake/lib/prserv/serv.py | |||
@@ -6,7 +6,7 @@ import queue | |||
6 | import socket | 6 | import socket |
7 | import io | 7 | import io |
8 | import sqlite3 | 8 | import sqlite3 |
9 | import bb.server.xmlrpc | 9 | import bb.server.xmlrpcclient |
10 | import prserv | 10 | import prserv |
11 | import prserv.db | 11 | import prserv.db |
12 | import errno | 12 | import errno |
@@ -300,7 +300,7 @@ class PRServerConnection(object): | |||
300 | host, port = singleton.getinfo() | 300 | host, port = singleton.getinfo() |
301 | self.host = host | 301 | self.host = host |
302 | self.port = port | 302 | self.port = port |
303 | self.connection, self.transport = bb.server.xmlrpc._create_server(self.host, self.port) | 303 | self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port) |
304 | 304 | ||
305 | def terminate(self): | 305 | def terminate(self): |
306 | try: | 306 | try: |