diff options
author | Etienne Cordonnier <ecordonnier@snap.com> | 2023-09-22 10:51:46 +0200 |
---|---|---|
committer | Steve Sakoman <steve@sakoman.com> | 2023-10-18 05:13:24 -1000 |
commit | 3a3afebf41077e097150bc33f3e292eed929ba11 (patch) | |
tree | 6e3ce05bef44a71f69398bca954a7959a974c69d /bitbake | |
parent | 222be3e3b989cb32cd8e3601615fac276c04c3ea (diff) | |
download | poky-3a3afebf41077e097150bc33f3e292eed929ba11.tar.gz |
bitbake: bitbake-worker/runqueue: Avoid unnecessary bytes object copies
declaring queue=b"" creates an object of types bytes().
bytes() is an immutable object, and therefore doing "self.queue = self.queue + r"
creates a new object containing "self.queue" concatenated with "r".
On my test setup, we are passing 180MB of data of "workerdata" to the bitbake-worker,
so those copies significantly slow down the initialization of the bitbake-worker.
Rather use bytearray() which a mutable type, and use extend() to avoid copies.
In my test setup, byterray.extend() is 10.000 times faster than copying the queue,
for a queue size of 180MB.
(Bitbake rev: 9993a89e5b97dda5f3657e5a7cc3a4fa94ff7111)
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Diffstat (limited to 'bitbake')
-rwxr-xr-x | bitbake/bin/bitbake-worker | 14 | ||||
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 4 |
2 files changed, 9 insertions, 9 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker index 5e62bc20de..e02f253205 100755 --- a/bitbake/bin/bitbake-worker +++ b/bitbake/bin/bitbake-worker | |||
@@ -91,19 +91,19 @@ def worker_fire_prepickled(event): | |||
91 | worker_thread_exit = False | 91 | worker_thread_exit = False |
92 | 92 | ||
93 | def worker_flush(worker_queue): | 93 | def worker_flush(worker_queue): |
94 | worker_queue_int = b"" | 94 | worker_queue_int = bytearray() |
95 | global worker_pipe, worker_thread_exit | 95 | global worker_pipe, worker_thread_exit |
96 | 96 | ||
97 | while True: | 97 | while True: |
98 | try: | 98 | try: |
99 | worker_queue_int = worker_queue_int + worker_queue.get(True, 1) | 99 | worker_queue_int.extend(worker_queue.get(True, 1)) |
100 | except queue.Empty: | 100 | except queue.Empty: |
101 | pass | 101 | pass |
102 | while (worker_queue_int or not worker_queue.empty()): | 102 | while (worker_queue_int or not worker_queue.empty()): |
103 | try: | 103 | try: |
104 | (_, ready, _) = select.select([], [worker_pipe], [], 1) | 104 | (_, ready, _) = select.select([], [worker_pipe], [], 1) |
105 | if not worker_queue.empty(): | 105 | if not worker_queue.empty(): |
106 | worker_queue_int = worker_queue_int + worker_queue.get() | 106 | worker_queue_int.extend(worker_queue.get()) |
107 | written = os.write(worker_pipe, worker_queue_int) | 107 | written = os.write(worker_pipe, worker_queue_int) |
108 | worker_queue_int = worker_queue_int[written:] | 108 | worker_queue_int = worker_queue_int[written:] |
109 | except (IOError, OSError) as e: | 109 | except (IOError, OSError) as e: |
@@ -338,12 +338,12 @@ class runQueueWorkerPipe(): | |||
338 | if pipeout: | 338 | if pipeout: |
339 | pipeout.close() | 339 | pipeout.close() |
340 | bb.utils.nonblockingfd(self.input) | 340 | bb.utils.nonblockingfd(self.input) |
341 | self.queue = b"" | 341 | self.queue = bytearray() |
342 | 342 | ||
343 | def read(self): | 343 | def read(self): |
344 | start = len(self.queue) | 344 | start = len(self.queue) |
345 | try: | 345 | try: |
346 | self.queue = self.queue + (self.input.read(102400) or b"") | 346 | self.queue.extend(self.input.read(102400) or b"") |
347 | except (OSError, IOError) as e: | 347 | except (OSError, IOError) as e: |
348 | if e.errno != errno.EAGAIN: | 348 | if e.errno != errno.EAGAIN: |
349 | raise | 349 | raise |
@@ -371,7 +371,7 @@ class BitbakeWorker(object): | |||
371 | def __init__(self, din): | 371 | def __init__(self, din): |
372 | self.input = din | 372 | self.input = din |
373 | bb.utils.nonblockingfd(self.input) | 373 | bb.utils.nonblockingfd(self.input) |
374 | self.queue = b"" | 374 | self.queue = bytearray() |
375 | self.cookercfg = None | 375 | self.cookercfg = None |
376 | self.databuilder = None | 376 | self.databuilder = None |
377 | self.data = None | 377 | self.data = None |
@@ -405,7 +405,7 @@ class BitbakeWorker(object): | |||
405 | if len(r) == 0: | 405 | if len(r) == 0: |
406 | # EOF on pipe, server must have terminated | 406 | # EOF on pipe, server must have terminated |
407 | self.sigterm_exception(signal.SIGTERM, None) | 407 | self.sigterm_exception(signal.SIGTERM, None) |
408 | self.queue = self.queue + r | 408 | self.queue.extend(r) |
409 | except (OSError, IOError): | 409 | except (OSError, IOError): |
410 | pass | 410 | pass |
411 | if len(self.queue): | 411 | if len(self.queue): |
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 569520707b..46ff30a7d3 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -3101,7 +3101,7 @@ class runQueuePipe(): | |||
3101 | if pipeout: | 3101 | if pipeout: |
3102 | pipeout.close() | 3102 | pipeout.close() |
3103 | bb.utils.nonblockingfd(self.input) | 3103 | bb.utils.nonblockingfd(self.input) |
3104 | self.queue = b"" | 3104 | self.queue = bytearray() |
3105 | self.d = d | 3105 | self.d = d |
3106 | self.rq = rq | 3106 | self.rq = rq |
3107 | self.rqexec = rqexec | 3107 | self.rqexec = rqexec |
@@ -3120,7 +3120,7 @@ class runQueuePipe(): | |||
3120 | 3120 | ||
3121 | start = len(self.queue) | 3121 | start = len(self.queue) |
3122 | try: | 3122 | try: |
3123 | self.queue = self.queue + (self.input.read(102400) or b"") | 3123 | self.queue.extend(self.input.read(102400) or b"") |
3124 | except (OSError, IOError) as e: | 3124 | except (OSError, IOError) as e: |
3125 | if e.errno != errno.EAGAIN: | 3125 | if e.errno != errno.EAGAIN: |
3126 | raise | 3126 | raise |