diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2013-08-31 23:40:55 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2013-09-01 15:51:10 +0100 |
commit | b306d7d9a4dd5dec0ed1ca91b50569f078ac103e (patch) | |
tree | 7c76ce19c1a731316ae58f1c2b270389673f2a55 /bitbake/lib/bb/server/xmlrpc.py | |
parent | d63e6a925a78f0f5fb7e7329b5cc2993f1ec9b14 (diff) | |
download | poky-b306d7d9a4dd5dec0ed1ca91b50569f078ac103e.tar.gz |
bitbake: server/process, server/xmlrpc, runqueue: Use select.select() on fds, not time.sleep()
The existing backend server implementations were inefficient since they
were sleeping for the full length of the timeouts rather than being woken when
there was data ready for them. It was assumed they would wake and perhaps did
when we forked processes directory but that is no longer the case.
This updates both the process and xmlrpc backends to wait using select(). This
does mean we need to pass the file descriptors to wait on from the internals
who know which these file descriptors are but this is a logical improvement.
Tests of a pathaolgical load on the process server of ~420 rapid tasks
executed on a server with BB_NUMBER_THREAD=48 went from a wall clock
measurement of the overall command execution time of 75s to a much more
reasonable 24s.
(Bitbake rev: 9bee497960889d9baa0a4284d79a384b18a8e826)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/server/xmlrpc.py')
-rw-r--r-- | bitbake/lib/bb/server/xmlrpc.py | 26 |
1 files changed, 12 insertions, 14 deletions
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py index 641e15e833..cca569d0e9 100644 --- a/bitbake/lib/bb/server/xmlrpc.py +++ b/bitbake/lib/bb/server/xmlrpc.py | |||
@@ -264,12 +264,9 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer): | |||
264 | Serve Requests. Overloaded to honor a quit command | 264 | Serve Requests. Overloaded to honor a quit command |
265 | """ | 265 | """ |
266 | self.quit = False | 266 | self.quit = False |
267 | self.timeout = 0 # Run Idle calls for our first callback | ||
268 | while not self.quit: | 267 | while not self.quit: |
269 | #print "Idle queue length %s" % len(self._idlefuns) | 268 | fds = [self] |
270 | self.handle_request() | 269 | nextsleep = 0.1 |
271 | #print "Idle timeout, running idle functions" | ||
272 | nextsleep = None | ||
273 | for function, data in self._idlefuns.items(): | 270 | for function, data in self._idlefuns.items(): |
274 | try: | 271 | try: |
275 | retval = function(self, data, False) | 272 | retval = function(self, data, False) |
@@ -277,21 +274,22 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer): | |||
277 | del self._idlefuns[function] | 274 | del self._idlefuns[function] |
278 | elif retval is True: | 275 | elif retval is True: |
279 | nextsleep = 0 | 276 | nextsleep = 0 |
280 | elif nextsleep is 0: | 277 | else: |
281 | continue | 278 | fds = fds + retval |
282 | elif nextsleep is None: | ||
283 | nextsleep = retval | ||
284 | elif retval < nextsleep: | ||
285 | nextsleep = retval | ||
286 | except SystemExit: | 279 | except SystemExit: |
287 | raise | 280 | raise |
288 | except: | 281 | except: |
289 | import traceback | 282 | import traceback |
290 | traceback.print_exc() | 283 | traceback.print_exc() |
291 | pass | 284 | pass |
292 | if nextsleep is None and len(self._idlefuns) > 0: | 285 | |
293 | nextsleep = 0 | 286 | socktimeout = self.socket.gettimeout() or nextsleep |
294 | self.timeout = nextsleep | 287 | socktimeout = min(socktimeout, nextsleep) |
288 | # Mirror what BaseServer handle_request would do | ||
289 | fd_sets = select.select(fds, [], [], socktimeout) | ||
290 | if fd_sets[0] and self in fd_sets[0]: | ||
291 | self._handle_request_noblock() | ||
292 | |||
295 | # Tell idle functions we're exiting | 293 | # Tell idle functions we're exiting |
296 | for function, data in self._idlefuns.items(): | 294 | for function, data in self._idlefuns.items(): |
297 | try: | 295 | try: |