summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-08-31 23:40:55 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-09-01 15:51:10 +0100
commitb306d7d9a4dd5dec0ed1ca91b50569f078ac103e (patch)
tree7c76ce19c1a731316ae58f1c2b270389673f2a55 /bitbake/lib/bb/runqueue.py
parentd63e6a925a78f0f5fb7e7329b5cc2993f1ec9b14 (diff)
downloadpoky-b306d7d9a4dd5dec0ed1ca91b50569f078ac103e.tar.gz
bitbake: server/process, server/xmlrpc, runqueue: Use select.select() on fds, not time.sleep()
The existing backend server implementations were inefficient since they were sleeping for the full length of the timeouts rather than being woken when there was data ready for them. It was assumed they would wake and perhaps did when we forked processes directory but that is no longer the case. This updates both the process and xmlrpc backends to wait using select(). This does mean we need to pass the file descriptors to wait on from the internals who know which these file descriptors are but this is a logical improvement. Tests of a pathaolgical load on the process server of ~420 rapid tasks executed on a server with BB_NUMBER_THREAD=48 went from a wall clock measurement of the overall command execution time of 75s to a much more reasonable 24s. (Bitbake rev: 9bee497960889d9baa0a4284d79a384b18a8e826) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py12
1 files changed, 10 insertions, 2 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 075c84985a..197308fd9d 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -895,6 +895,14 @@ class RunQueue:
895 if self.fakeworkerpipe: 895 if self.fakeworkerpipe:
896 self.fakeworkerpipe.read() 896 self.fakeworkerpipe.read()
897 897
898 def active_fds(self):
899 fds = []
900 if self.workerpipe:
901 fds.append(self.workerpipe.input)
902 if self.fakeworkerpipe:
903 fds.append(self.fakeworkerpipe.input)
904 return fds
905
898 def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): 906 def check_stamp_task(self, task, taskname = None, recurse = False, cache = None):
899 def get_timestamp(f): 907 def get_timestamp(f):
900 try: 908 try:
@@ -972,7 +980,7 @@ class RunQueue:
972 (if the abort on failure configuration option isn't set) 980 (if the abort on failure configuration option isn't set)
973 """ 981 """
974 982
975 retval = 0.5 983 retval = True
976 984
977 if self.state is runQueuePrepare: 985 if self.state is runQueuePrepare:
978 self.rqexe = RunQueueExecuteDummy(self) 986 self.rqexe = RunQueueExecuteDummy(self)
@@ -1375,7 +1383,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1375 1383
1376 if self.stats.active > 0: 1384 if self.stats.active > 0:
1377 self.rq.read_workers() 1385 self.rq.read_workers()
1378 return 0.5 1386 return self.rq.active_fds()
1379 1387
1380 if len(self.failed_fnids) != 0: 1388 if len(self.failed_fnids) != 0:
1381 self.rq.state = runQueueFailed 1389 self.rq.state = runQueueFailed