summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/utils.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2017-09-01 13:24:31 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-09-01 23:52:10 (GMT)
commit627d9a71dfedd30da2beb5e967f38b2f908f98d5 (patch)
treef58bad6b3e0204d467623deea74a6084bdea24ec /meta/lib/oe/utils.py
parent55f0493e26881123c6317edfb9dc7af738d3d2d5 (diff)
downloadpoky-627d9a71dfedd30da2beb5e967f38b2f908f98d5.tar.gz
lib/oe/utils: Handle exceptions in multiprocess_exec
Currently exceptions that happen in pool commands are ignored. Any errors would be printed on the console but everything else is silent. Switch to use pool.map_async which allows for an error_callback which we can use to detect exceptions and make sure these errors are handled. (From OE-Core rev: 7f2f9b3ff011b340b5d23bb7c47b12c357dc9f02) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe/utils.py')
-rw-r--r--meta/lib/oe/utils.py21
1 files changed, 13 insertions, 8 deletions
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 822d0cd..643ab78 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -224,25 +224,30 @@ def multiprocess_exec(commands, function):
224 def init_worker(): 224 def init_worker():
225 signal.signal(signal.SIGINT, signal.SIG_IGN) 225 signal.signal(signal.SIGINT, signal.SIG_IGN)
226 226
227 fails = []
228
229 def failures(res):
230 fails.append(res)
231
227 nproc = min(multiprocessing.cpu_count(), len(commands)) 232 nproc = min(multiprocessing.cpu_count(), len(commands))
228 pool = bb.utils.multiprocessingpool(nproc, init_worker) 233 pool = bb.utils.multiprocessingpool(nproc, init_worker)
229 imap = pool.imap(function, commands)
230 234
231 try: 235 try:
232 res = list(imap) 236 mapresult = pool.map_async(function, commands, error_callback=failures)
237
233 pool.close() 238 pool.close()
234 pool.join() 239 pool.join()
235 results = [] 240 results = mapresult.get()
236 for result in res:
237 if result is not None:
238 results.append(result)
239 return results
240
241 except KeyboardInterrupt: 241 except KeyboardInterrupt:
242 pool.terminate() 242 pool.terminate()
243 pool.join() 243 pool.join()
244 raise 244 raise
245 245
246 if fails:
247 raise fails[0]
248
249 return results
250
246def squashspaces(string): 251def squashspaces(string):
247 import re 252 import re
248 return re.sub("\s+", " ", string).strip() 253 return re.sub("\s+", " ", string).strip()