summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/lib/oe/package_manager.py32
-rw-r--r--meta/lib/oe/utils.py29
2 files changed, 33 insertions, 28 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 8be3d41706..f8fc3c28bf 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -7,6 +7,7 @@ import multiprocessing
7import re 7import re
8import bb 8import bb
9import tempfile 9import tempfile
10import oe.utils
10 11
11 12
12# this can be used by all PM backends to create the index files in parallel 13# this can be used by all PM backends to create the index files in parallel
@@ -116,16 +117,7 @@ class RpmIndexer(Indexer):
116 bb.note("There are no packages in %s" % self.deploy_dir) 117 bb.note("There are no packages in %s" % self.deploy_dir)
117 return 118 return
118 119
119 nproc = multiprocessing.cpu_count() 120 oe.utils.multiprocess_exec(index_cmds, create_index)
120 pool = bb.utils.multiprocessingpool(nproc)
121 results = list(pool.imap(create_index, index_cmds))
122 pool.close()
123 pool.join()
124
125 for result in results:
126 if result is not None:
127 return(result)
128
129 121
130class OpkgIndexer(Indexer): 122class OpkgIndexer(Indexer):
131 def write_index(self): 123 def write_index(self):
@@ -161,15 +153,7 @@ class OpkgIndexer(Indexer):
161 bb.note("There are no packages in %s!" % self.deploy_dir) 153 bb.note("There are no packages in %s!" % self.deploy_dir)
162 return 154 return
163 155
164 nproc = multiprocessing.cpu_count() 156 oe.utils.multiprocess_exec(index_cmds, create_index)
165 pool = bb.utils.multiprocessingpool(nproc)
166 results = list(pool.imap(create_index, index_cmds))
167 pool.close()
168 pool.join()
169
170 for result in results:
171 if result is not None:
172 return(result)
173 157
174 158
175class DpkgIndexer(Indexer): 159class DpkgIndexer(Indexer):
@@ -210,15 +194,7 @@ class DpkgIndexer(Indexer):
210 bb.note("There are no packages in %s" % self.deploy_dir) 194 bb.note("There are no packages in %s" % self.deploy_dir)
211 return 195 return
212 196
213 nproc = multiprocessing.cpu_count() 197 oe.utils.multiprocess_exec(index_cmds, create_index)
214 pool = bb.utils.multiprocessingpool(nproc)
215 results = list(pool.imap(create_index, index_cmds))
216 pool.close()
217 pool.join()
218
219 for result in results:
220 if result is not None:
221 return(result)
222 198
223 199
224class PkgsList(object): 200class PkgsList(object):
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 0a1d1080c9..92e21a4e0e 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -151,3 +151,32 @@ def execute_pre_post_process(d, cmds):
151 if cmd != '': 151 if cmd != '':
152 bb.note("Executing %s ..." % cmd) 152 bb.note("Executing %s ..." % cmd)
153 bb.build.exec_func(cmd, d) 153 bb.build.exec_func(cmd, d)
154
155def multiprocess_exec(commands, function):
156 import signal
157 import multiprocessing
158
159 if not commands:
160 return []
161
162 def init_worker():
163 signal.signal(signal.SIGINT, signal.SIG_IGN)
164
165 nproc = min(multiprocessing.cpu_count(), len(commands))
166 pool = bb.utils.multiprocessingpool(nproc, init_worker)
167 imap = pool.imap(function, commands)
168
169 try:
170 results = list(imap)
171 pool.close()
172 pool.join()
173 results = []
174 for result in results:
175 if result is not None:
176 results.append(result)
177 return results
178
179 except KeyboardInterrupt:
180 pool.terminate()
181 pool.join()
182 raise