summaryrefslogtreecommitdiffstats
path: root/bitbake/bin/bitbake-worker
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2016-12-07 12:04:43 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-12-08 10:32:21 +0000
commit24b668efa0d7bb277045b4361ad1676175c5a2cc (patch)
tree3338f04a32b48aaf822ea757cf8d390093c5a4c1 /bitbake/bin/bitbake-worker
parenteb937ee0a1ea1e695aa995c2d1a3561ba12335f8 (diff)
downloadpoky-24b668efa0d7bb277045b4361ad1676175c5a2cc.tar.gz
bitbake: bitbake-worker: Further IO performance tweaks
Looking further at the CPU loads on systems running large numbers of tasks, the following things helps performance: * Loop on waitpid until there are no processes still waiting * Using select to wait for the cooker pipe to be writable before writing avoiding pointless 100% cpu usage * Only reading from worker pipes that select highlights are readable (Bitbake rev: 9375349e27b08b4d1cfe4825c042d4c82120e00b) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/bin/bitbake-worker')
-rwxr-xr-xbitbake/bin/bitbake-worker13
1 files changed, 9 insertions, 4 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 10de54c9c3..97b32c3878 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -95,6 +95,7 @@ def worker_flush(worker_queue):
95 pass 95 pass
96 while (worker_queue_int or not worker_queue.empty()): 96 while (worker_queue_int or not worker_queue.empty()):
97 try: 97 try:
98 (_, ready, _) = select.select([], [worker_pipe], [], 1)
98 if not worker_queue.empty(): 99 if not worker_queue.empty():
99 worker_queue_int = worker_queue_int + worker_queue.get() 100 worker_queue_int = worker_queue_int + worker_queue.get()
100 written = os.write(worker_pipe, worker_queue_int) 101 written = os.write(worker_pipe, worker_queue_int)
@@ -369,9 +370,11 @@ class BitbakeWorker(object):
369 self.handle_item(b"quit", self.handle_quit) 370 self.handle_item(b"quit", self.handle_quit)
370 371
371 for pipe in self.build_pipes: 372 for pipe in self.build_pipes:
372 self.build_pipes[pipe].read() 373 if self.build_pipes[pipe].input in ready:
374 self.build_pipes[pipe].read()
373 if len(self.build_pids): 375 if len(self.build_pids):
374 self.process_waitpid() 376 while self.process_waitpid():
377 continue
375 378
376 379
377 def handle_item(self, item, func): 380 def handle_item(self, item, func):
@@ -426,9 +429,9 @@ class BitbakeWorker(object):
426 try: 429 try:
427 pid, status = os.waitpid(-1, os.WNOHANG) 430 pid, status = os.waitpid(-1, os.WNOHANG)
428 if pid == 0 or os.WIFSTOPPED(status): 431 if pid == 0 or os.WIFSTOPPED(status):
429 return None 432 return False
430 except OSError: 433 except OSError:
431 return None 434 return False
432 435
433 workerlog_write("Exit code of %s for pid %s\n" % (status, pid)) 436 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
434 437
@@ -447,6 +450,8 @@ class BitbakeWorker(object):
447 450
448 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>") 451 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
449 452
453 return True
454
450 def handle_finishnow(self, _): 455 def handle_finishnow(self, _):
451 if self.build_pids: 456 if self.build_pids:
452 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids)) 457 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))