From 2365e00b7c5e78f2ce5043a2e2495fef346ecb3d Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 24 Dec 2018 16:34:32 +0000 Subject: bitbake: process: Rewrite multiple connection handling If the bitbake server recieved multiple connections, it currently closes ones it can't handle (while its dealing with another). This is rather antisocial behaviour which causes clients to quickly run through their retries and abort. Instead, queue any other connections until the current one is closed. This way the client can decide when it wants to stop waiting for the server. If the client is gone by the time we handle it, we handle that gracefully. This also fixes a number of bugs in the connection handling where connections which did drop early were badly handled causing tracebacks in the logs. Also, handle queue incomming connections in a loop to ensure that the main client handling doesn't starve that piece of the system. This code was stress tested by running 50 connection attempts in parallel at once, ensuring the code correctly handled them. (Bitbake rev: 02845a561b38658ac3edf5cc9c34625ed860d34f) Signed-off-by: Richard Purdie --- bitbake/lib/bb/server/process.py | 56 ++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 22 deletions(-) (limited to 'bitbake/lib/bb/server') diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index 1e4b51e35e..28b8eb9b86 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py @@ -130,6 +130,7 @@ class ProcessServer(multiprocessing.Process): bb.utils.set_process_name("Cooker") ready = [] + newconnections = [] self.controllersock = False fds = [self.sock] @@ -138,37 +139,48 @@ class ProcessServer(multiprocessing.Process): print("Entering server connection loop") def disconnect_client(self, fds): - if not self.haveui: - return print("Disconnecting Client") - fds.remove(self.controllersock) - fds.remove(self.command_channel) - bb.event.unregister_UIHhandler(self.event_handle, True) - self.command_channel_reply.writer.close() - self.event_writer.writer.close() - del self.event_writer - self.controllersock.close() - self.controllersock = False - self.haveui = False - self.lastui = time.time() - self.cooker.clientComplete() - if self.timeout is None: + if self.controllersock: + fds.remove(self.controllersock) + self.controllersock.close() + self.controllersock = False + if self.haveui: + fds.remove(self.command_channel) + bb.event.unregister_UIHhandler(self.event_handle, True) + self.command_channel_reply.writer.close() + self.event_writer.writer.close() + self.command_channel.close() + self.command_channel = False + del self.event_writer + self.lastui = time.time() + self.cooker.clientComplete() + self.haveui = False + ready = select.select(fds,[],[],0)[0] + if newconnections: + print("Starting new client") + conn = newconnections.pop(-1) + fds.append(conn) + self.controllersock = conn + elif self.timeout is None and not ready: print("No timeout, exiting.") self.quit = True while not self.quit: if self.sock in ready: - self.controllersock, address = self.sock.accept() - if self.haveui: - print("Dropping connection attempt as we have a UI %s" % (str(ready))) - self.controllersock.close() - else: - print("Accepting %s" % (str(ready))) - fds.append(self.controllersock) + while select.select([self.sock],[],[],0)[0]: + controllersock, address = self.sock.accept() + if self.controllersock: + print("Queuing %s (%s)" % (str(ready), str(newconnections))) + newconnections.append(controllersock) + else: + print("Accepting %s (%s)" % (str(ready), str(newconnections))) + self.controllersock = controllersock + fds.append(controllersock) if self.controllersock in ready: try: - print("Connecting Client") + print("Processing Client") ui_fds = recvfds(self.controllersock, 3) + print("Connecting Client") # Where to write events to writer = ConnectionWriter(ui_fds[0]) -- cgit v1.2.3-54-g00ecf