From b306d7d9a4dd5dec0ed1ca91b50569f078ac103e Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Sat, 31 Aug 2013 23:40:55 +0100 Subject: bitbake: server/process, server/xmlrpc, runqueue: Use select.select() on fds, not time.sleep() The existing backend server implementations were inefficient since they were sleeping for the full length of the timeouts rather than being woken when there was data ready for them. It was assumed they would wake and perhaps did when we forked processes directory but that is no longer the case. This updates both the process and xmlrpc backends to wait using select(). This does mean we need to pass the file descriptors to wait on from the internals who know which these file descriptors are but this is a logical improvement. Tests of a pathaolgical load on the process server of ~420 rapid tasks executed on a server with BB_NUMBER_THREAD=48 went from a wall clock measurement of the overall command execution time of 75s to a much more reasonable 24s. (Bitbake rev: 9bee497960889d9baa0a4284d79a384b18a8e826) Signed-off-by: Richard Purdie --- bitbake/lib/bb/server/process.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'bitbake/lib/bb/server/process.py') diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index e2cec49b74..c0af052ebd 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py @@ -29,6 +29,7 @@ import os import signal import sys import time +import select from Queue import Empty from multiprocessing import Event, Process, util, Queue, Pipe, queues @@ -105,7 +106,7 @@ class ProcessServer(Process, BaseImplServer): command = self.command_channel.recv() self.runCommand(command) - self.idle_commands(.1) + self.idle_commands(.1, [self.event_queue._reader, self.command_channel]) except Exception: logger.exception('Running command %s', command) @@ -115,7 +116,7 @@ class ProcessServer(Process, BaseImplServer): self.cooker.stop() self.idle_commands(.1) - def idle_commands(self, delay): + def idle_commands(self, delay, fds = []): nextsleep = delay for function, data in self._idlefuns.items(): @@ -127,15 +128,15 @@ class ProcessServer(Process, BaseImplServer): nextsleep = None elif nextsleep is None: continue - elif retval < nextsleep: - nextsleep = retval + else: + fds = fds + retval except SystemExit: raise except Exception: logger.exception('Running idle function') if nextsleep is not None: - time.sleep(nextsleep) + select.select(fds,[],[],nextsleep) def runCommand(self, command): """ -- cgit v1.2.3-54-g00ecf