summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2014-08-21 21:46:28 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-08-23 09:31:41 +0100
commitbe1b198076cd8849ab6ecc16ad08556c5981f3d9 (patch)
tree3d6c045cbe857d4d305fedf0cf6d69ea5507b5ec /meta/lib
parent6ab380ddde11a8695ae50efb3710c72fb8a772ba (diff)
downloadpoky-be1b198076cd8849ab6ecc16ad08556c5981f3d9.tar.gz
lib/oe/utils: Add utils function for multiprocess execution
Our usage of multitprocessing is problematic. In particular, there is a bug in python 2.7 multiprocessing where signals are not handled until command completion instead of immediately. This factors the multiprocess code into a function which is enhanced with a workaround to ensure immediate signal handling and also better SIGINT handling which should happen in the parent, not the children to ensure clean exits. The workaround for the signals is being added to the core bb.utils function so it can benefit all users. package_manager is then converted to use the new code. (From OE-Core rev: 72d153a3a90d31d9f4e41d77da24e44ccb33c56e) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oe/package_manager.py32
-rw-r--r--meta/lib/oe/utils.py29
2 files changed, 33 insertions, 28 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 8be3d41706..f8fc3c28bf 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -7,6 +7,7 @@ import multiprocessing
7import re 7import re
8import bb 8import bb
9import tempfile 9import tempfile
10import oe.utils
10 11
11 12
12# this can be used by all PM backends to create the index files in parallel 13# this can be used by all PM backends to create the index files in parallel
@@ -116,16 +117,7 @@ class RpmIndexer(Indexer):
116 bb.note("There are no packages in %s" % self.deploy_dir) 117 bb.note("There are no packages in %s" % self.deploy_dir)
117 return 118 return
118 119
119 nproc = multiprocessing.cpu_count() 120 oe.utils.multiprocess_exec(index_cmds, create_index)
120 pool = bb.utils.multiprocessingpool(nproc)
121 results = list(pool.imap(create_index, index_cmds))
122 pool.close()
123 pool.join()
124
125 for result in results:
126 if result is not None:
127 return(result)
128
129 121
130class OpkgIndexer(Indexer): 122class OpkgIndexer(Indexer):
131 def write_index(self): 123 def write_index(self):
@@ -161,15 +153,7 @@ class OpkgIndexer(Indexer):
161 bb.note("There are no packages in %s!" % self.deploy_dir) 153 bb.note("There are no packages in %s!" % self.deploy_dir)
162 return 154 return
163 155
164 nproc = multiprocessing.cpu_count() 156 oe.utils.multiprocess_exec(index_cmds, create_index)
165 pool = bb.utils.multiprocessingpool(nproc)
166 results = list(pool.imap(create_index, index_cmds))
167 pool.close()
168 pool.join()
169
170 for result in results:
171 if result is not None:
172 return(result)
173 157
174 158
175class DpkgIndexer(Indexer): 159class DpkgIndexer(Indexer):
@@ -210,15 +194,7 @@ class DpkgIndexer(Indexer):
210 bb.note("There are no packages in %s" % self.deploy_dir) 194 bb.note("There are no packages in %s" % self.deploy_dir)
211 return 195 return
212 196
213 nproc = multiprocessing.cpu_count() 197 oe.utils.multiprocess_exec(index_cmds, create_index)
214 pool = bb.utils.multiprocessingpool(nproc)
215 results = list(pool.imap(create_index, index_cmds))
216 pool.close()
217 pool.join()
218
219 for result in results:
220 if result is not None:
221 return(result)
222 198
223 199
224class PkgsList(object): 200class PkgsList(object):
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 0a1d1080c9..92e21a4e0e 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -151,3 +151,32 @@ def execute_pre_post_process(d, cmds):
151 if cmd != '': 151 if cmd != '':
152 bb.note("Executing %s ..." % cmd) 152 bb.note("Executing %s ..." % cmd)
153 bb.build.exec_func(cmd, d) 153 bb.build.exec_func(cmd, d)
154
155def multiprocess_exec(commands, function):
156 import signal
157 import multiprocessing
158
159 if not commands:
160 return []
161
162 def init_worker():
163 signal.signal(signal.SIGINT, signal.SIG_IGN)
164
165 nproc = min(multiprocessing.cpu_count(), len(commands))
166 pool = bb.utils.multiprocessingpool(nproc, init_worker)
167 imap = pool.imap(function, commands)
168
169 try:
170 results = list(imap)
171 pool.close()
172 pool.join()
173 results = []
174 for result in results:
175 if result is not None:
176 results.append(result)
177 return results
178
179 except KeyboardInterrupt:
180 pool.terminate()
181 pool.join()
182 raise