summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristopher Larson <chris_larson@mentor.com>2012-01-08 23:21:27 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-01-20 16:47:38 +0000
commit0779a90e2a39a16867afb1ce0202db13f62d4f77 (patch)
treeff9b55ea6f54df46ba488a371089e47f46258f43
parent299fa3489b6c1ed0e7dc5df6c297639934db8edb (diff)
downloadpoky-0779a90e2a39a16867afb1ce0202db13f62d4f77.tar.gz
cooker: use futures rather than a multiprocessing pool
This avoids some silent parser hangs we were seeing which were near impossible to debug as no user feedback was given. [RP: Tweak commit message] (Bitbake rev: d104f29871c04a5a36600a35b2568b49e5b21ca0) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py29
1 files changed, 18 insertions, 11 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 4197a02314..cbac1f73e0 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -32,6 +32,7 @@ import sre_constants
32import threading 32import threading
33from cStringIO import StringIO 33from cStringIO import StringIO
34from contextlib import closing 34from contextlib import closing
35from concurrent import futures
35from functools import wraps 36from functools import wraps
36from collections import defaultdict 37from collections import defaultdict
37import bb, bb.exceptions, bb.command 38import bb, bb.exceptions, bb.command
@@ -1462,20 +1463,16 @@ class CookerParser(object):
1462 self.start() 1463 self.start()
1463 1464
1464 def start(self): 1465 def start(self):
1465 def init(cfg):
1466 parse_file.cfg = cfg
1467 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
1468
1469 self.results = self.load_cached() 1466 self.results = self.load_cached()
1470 1467
1471 if self.toparse: 1468 if self.toparse:
1472 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) 1469 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
1473 1470
1474 self.pool = multiprocessing.Pool(self.num_processes, init, [self.cfgdata]) 1471 parse_file.cfg = self.cfgdata
1475 parsed = self.pool.imap(parse_file, self.willparse) 1472 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cfgdata,), exitpriority=1)
1476 self.pool.close() 1473 self.executor = futures.ProcessPoolExecutor(max_workers=self.num_processes)
1477 1474 self.futures = dict((self.executor.submit(parse_file, task), task) for task in self.willparse)
1478 self.results = itertools.chain(self.results, parsed) 1475 self.results = itertools.chain(self.results, self.parse_gen())
1479 1476
1480 def shutdown(self, clean=True): 1477 def shutdown(self, clean=True):
1481 if not self.toparse: 1478 if not self.toparse:
@@ -1488,8 +1485,9 @@ class CookerParser(object):
1488 self.total) 1485 self.total)
1489 bb.event.fire(event, self.cfgdata) 1486 bb.event.fire(event, self.cfgdata)
1490 else: 1487 else:
1491 self.pool.terminate() 1488 for future in self.futures:
1492 self.pool.join() 1489 future.cancel()
1490 self.executor.shutdown()
1493 1491
1494 sync = threading.Thread(target=self.bb_cache.sync) 1492 sync = threading.Thread(target=self.bb_cache.sync)
1495 sync.start() 1493 sync.start()
@@ -1501,6 +1499,15 @@ class CookerParser(object):
1501 cached, infos = self.bb_cache.load(filename, appends, self.cfgdata) 1499 cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
1502 yield not cached, infos 1500 yield not cached, infos
1503 1501
1502 def parse_gen(self):
1503 for future in futures.as_completed(self.futures):
1504 task = self.futures[future]
1505 exc = future.exception()
1506 if exc:
1507 raise exc
1508 else:
1509 yield future.result()
1510
1504 def parse_next(self): 1511 def parse_next(self):
1505 try: 1512 try:
1506 parsed, result = self.results.next() 1513 parsed, result = self.results.next()