diff options
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/compat.py | 926 | ||||
-rw-r--r-- | bitbake/lib/bb/server/process.py | 46 | ||||
-rw-r--r-- | bitbake/lib/bb/server/xmlrpc.py | 104 | ||||
-rw-r--r-- | bitbake/lib/bb/utils.py | 7 |
4 files changed, 15 insertions, 1068 deletions
diff --git a/bitbake/lib/bb/compat.py b/bitbake/lib/bb/compat.py index 440a2fbc8b..de1923d28a 100644 --- a/bitbake/lib/bb/compat.py +++ b/bitbake/lib/bb/compat.py | |||
@@ -1,928 +1,6 @@ | |||
1 | """Code pulled from future python versions, here for compatibility""" | 1 | """Code pulled from future python versions, here for compatibility""" |
2 | 2 | ||
3 | from collections import MutableMapping, KeysView, ValuesView, ItemsView | 3 | from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict |
4 | try: | 4 | from functools import total_ordering |
5 | from thread import get_ident as _get_ident | ||
6 | except ImportError: | ||
7 | from dummy_thread import get_ident as _get_ident | ||
8 | |||
9 | def total_ordering(cls): | ||
10 | """Class decorator that fills in missing ordering methods""" | ||
11 | convert = { | ||
12 | '__lt__': [('__gt__', lambda self, other: other < self), | ||
13 | ('__le__', lambda self, other: not other < self), | ||
14 | ('__ge__', lambda self, other: not self < other)], | ||
15 | '__le__': [('__ge__', lambda self, other: other <= self), | ||
16 | ('__lt__', lambda self, other: not other <= self), | ||
17 | ('__gt__', lambda self, other: not self <= other)], | ||
18 | '__gt__': [('__lt__', lambda self, other: other > self), | ||
19 | ('__ge__', lambda self, other: not other > self), | ||
20 | ('__le__', lambda self, other: not self > other)], | ||
21 | '__ge__': [('__le__', lambda self, other: other >= self), | ||
22 | ('__gt__', lambda self, other: not other >= self), | ||
23 | ('__lt__', lambda self, other: not self >= other)] | ||
24 | } | ||
25 | roots = set(dir(cls)) & set(convert) | ||
26 | if not roots: | ||
27 | raise ValueError('must define at least one ordering operation: < > <= >=') | ||
28 | root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ | ||
29 | for opname, opfunc in convert[root]: | ||
30 | if opname not in roots: | ||
31 | opfunc.__name__ = opname | ||
32 | opfunc.__doc__ = getattr(int, opname).__doc__ | ||
33 | setattr(cls, opname, opfunc) | ||
34 | return cls | ||
35 | |||
36 | class OrderedDict(dict): | ||
37 | 'Dictionary that remembers insertion order' | ||
38 | # An inherited dict maps keys to values. | ||
39 | # The inherited dict provides __getitem__, __len__, __contains__, and get. | ||
40 | # The remaining methods are order-aware. | ||
41 | # Big-O running times for all methods are the same as regular dictionaries. | ||
42 | |||
43 | # The internal self.__map dict maps keys to links in a doubly linked list. | ||
44 | # The circular doubly linked list starts and ends with a sentinel element. | ||
45 | # The sentinel element never gets deleted (this simplifies the algorithm). | ||
46 | # Each link is stored as a list of length three: [PREV, NEXT, KEY]. | ||
47 | |||
48 | def __init__(self, *args, **kwds): | ||
49 | '''Initialize an ordered dictionary. The signature is the same as | ||
50 | regular dictionaries, but keyword arguments are not recommended because | ||
51 | their insertion order is arbitrary. | ||
52 | |||
53 | ''' | ||
54 | if len(args) > 1: | ||
55 | raise TypeError('expected at most 1 arguments, got %d' % len(args)) | ||
56 | try: | ||
57 | self.__root | ||
58 | except AttributeError: | ||
59 | self.__root = root = [] # sentinel node | ||
60 | root[:] = [root, root, None] | ||
61 | self.__map = {} | ||
62 | self.__update(*args, **kwds) | ||
63 | |||
64 | def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): | ||
65 | 'od.__setitem__(i, y) <==> od[i]=y' | ||
66 | # Setting a new item creates a new link at the end of the linked list, | ||
67 | # and the inherited dictionary is updated with the new key/value pair. | ||
68 | if key not in self: | ||
69 | root = self.__root | ||
70 | last = root[PREV] | ||
71 | last[NEXT] = root[PREV] = self.__map[key] = [last, root, key] | ||
72 | dict_setitem(self, key, value) | ||
73 | |||
74 | def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): | ||
75 | 'od.__delitem__(y) <==> del od[y]' | ||
76 | # Deleting an existing item uses self.__map to find the link which gets | ||
77 | # removed by updating the links in the predecessor and successor nodes. | ||
78 | dict_delitem(self, key) | ||
79 | link_prev, link_next, key = self.__map.pop(key) | ||
80 | link_prev[NEXT] = link_next | ||
81 | link_next[PREV] = link_prev | ||
82 | |||
83 | def __iter__(self): | ||
84 | 'od.__iter__() <==> iter(od)' | ||
85 | # Traverse the linked list in order. | ||
86 | NEXT, KEY = 1, 2 | ||
87 | root = self.__root | ||
88 | curr = root[NEXT] | ||
89 | while curr is not root: | ||
90 | yield curr[KEY] | ||
91 | curr = curr[NEXT] | ||
92 | |||
93 | def __reversed__(self): | ||
94 | 'od.__reversed__() <==> reversed(od)' | ||
95 | # Traverse the linked list in reverse order. | ||
96 | PREV, KEY = 0, 2 | ||
97 | root = self.__root | ||
98 | curr = root[PREV] | ||
99 | while curr is not root: | ||
100 | yield curr[KEY] | ||
101 | curr = curr[PREV] | ||
102 | |||
103 | def clear(self): | ||
104 | 'od.clear() -> None. Remove all items from od.' | ||
105 | for node in self.__map.itervalues(): | ||
106 | del node[:] | ||
107 | root = self.__root | ||
108 | root[:] = [root, root, None] | ||
109 | self.__map.clear() | ||
110 | dict.clear(self) | ||
111 | |||
112 | # -- the following methods do not depend on the internal structure -- | ||
113 | |||
114 | def keys(self): | ||
115 | 'od.keys() -> list of keys in od' | ||
116 | return list(self) | ||
117 | |||
118 | def values(self): | ||
119 | 'od.values() -> list of values in od' | ||
120 | return [self[key] for key in self] | ||
121 | |||
122 | def items(self): | ||
123 | 'od.items() -> list of (key, value) pairs in od' | ||
124 | return [(key, self[key]) for key in self] | ||
125 | |||
126 | def iterkeys(self): | ||
127 | 'od.iterkeys() -> an iterator over the keys in od' | ||
128 | return iter(self) | ||
129 | |||
130 | def itervalues(self): | ||
131 | 'od.itervalues -> an iterator over the values in od' | ||
132 | for k in self: | ||
133 | yield self[k] | ||
134 | |||
135 | def iteritems(self): | ||
136 | 'od.iteritems -> an iterator over the (key, value) pairs in od' | ||
137 | for k in self: | ||
138 | yield (k, self[k]) | ||
139 | |||
140 | update = MutableMapping.update | ||
141 | |||
142 | __update = update # let subclasses override update without breaking __init__ | ||
143 | |||
144 | __marker = object() | ||
145 | |||
146 | def pop(self, key, default=__marker): | ||
147 | '''od.pop(k[,d]) -> v, remove specified key and return the corresponding | ||
148 | value. If key is not found, d is returned if given, otherwise KeyError | ||
149 | is raised. | ||
150 | |||
151 | ''' | ||
152 | if key in self: | ||
153 | result = self[key] | ||
154 | del self[key] | ||
155 | return result | ||
156 | if default is self.__marker: | ||
157 | raise KeyError(key) | ||
158 | return default | ||
159 | |||
160 | def setdefault(self, key, default=None): | ||
161 | 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' | ||
162 | if key in self: | ||
163 | return self[key] | ||
164 | self[key] = default | ||
165 | return default | ||
166 | |||
167 | def popitem(self, last=True): | ||
168 | '''od.popitem() -> (k, v), return and remove a (key, value) pair. | ||
169 | Pairs are returned in LIFO order if last is true or FIFO order if false. | ||
170 | |||
171 | ''' | ||
172 | if not self: | ||
173 | raise KeyError('dictionary is empty') | ||
174 | key = next(reversed(self) if last else iter(self)) | ||
175 | value = self.pop(key) | ||
176 | return key, value | ||
177 | |||
178 | def __repr__(self, _repr_running={}): | ||
179 | 'od.__repr__() <==> repr(od)' | ||
180 | call_key = id(self), _get_ident() | ||
181 | if call_key in _repr_running: | ||
182 | return '...' | ||
183 | _repr_running[call_key] = 1 | ||
184 | try: | ||
185 | if not self: | ||
186 | return '%s()' % (self.__class__.__name__,) | ||
187 | return '%s(%r)' % (self.__class__.__name__, self.items()) | ||
188 | finally: | ||
189 | del _repr_running[call_key] | ||
190 | |||
191 | def __reduce__(self): | ||
192 | 'Return state information for pickling' | ||
193 | items = [[k, self[k]] for k in self] | ||
194 | inst_dict = vars(self).copy() | ||
195 | for k in vars(OrderedDict()): | ||
196 | inst_dict.pop(k, None) | ||
197 | if inst_dict: | ||
198 | return (self.__class__, (items,), inst_dict) | ||
199 | return self.__class__, (items,) | ||
200 | |||
201 | def copy(self): | ||
202 | 'od.copy() -> a shallow copy of od' | ||
203 | return self.__class__(self) | ||
204 | |||
205 | @classmethod | ||
206 | def fromkeys(cls, iterable, value=None): | ||
207 | '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. | ||
208 | If not specified, the value defaults to None. | ||
209 | |||
210 | ''' | ||
211 | self = cls() | ||
212 | for key in iterable: | ||
213 | self[key] = value | ||
214 | return self | ||
215 | |||
216 | def __eq__(self, other): | ||
217 | '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive | ||
218 | while comparison to a regular mapping is order-insensitive. | ||
219 | |||
220 | ''' | ||
221 | if isinstance(other, OrderedDict): | ||
222 | return len(self)==len(other) and self.items() == other.items() | ||
223 | return dict.__eq__(self, other) | ||
224 | |||
225 | def __ne__(self, other): | ||
226 | 'od.__ne__(y) <==> od!=y' | ||
227 | return not self == other | ||
228 | |||
229 | # -- the following methods support python 3.x style dictionary views -- | ||
230 | |||
231 | def viewkeys(self): | ||
232 | "od.viewkeys() -> a set-like object providing a view on od's keys" | ||
233 | return KeysView(self) | ||
234 | |||
235 | def viewvalues(self): | ||
236 | "od.viewvalues() -> an object providing a view on od's values" | ||
237 | return ValuesView(self) | ||
238 | |||
239 | def viewitems(self): | ||
240 | "od.viewitems() -> a set-like object providing a view on od's items" | ||
241 | return ItemsView(self) | ||
242 | |||
243 | # Multiprocessing pool code imported from python 2.7.3. Previous versions of | ||
244 | # python have issues in this code which hang pool usage | ||
245 | |||
246 | # | ||
247 | # Module providing the `Pool` class for managing a process pool | ||
248 | # | ||
249 | # multiprocessing/pool.py | ||
250 | # | ||
251 | # Copyright (c) 2006-2008, R Oudkerk | ||
252 | # All rights reserved. | ||
253 | # | ||
254 | # Redistribution and use in source and binary forms, with or without | ||
255 | # modification, are permitted provided that the following conditions | ||
256 | # are met: | ||
257 | # | ||
258 | # 1. Redistributions of source code must retain the above copyright | ||
259 | # notice, this list of conditions and the following disclaimer. | ||
260 | # 2. Redistributions in binary form must reproduce the above copyright | ||
261 | # notice, this list of conditions and the following disclaimer in the | ||
262 | # documentation and/or other materials provided with the distribution. | ||
263 | # 3. Neither the name of author nor the names of any contributors may be | ||
264 | # used to endorse or promote products derived from this software | ||
265 | # without specific prior written permission. | ||
266 | # | ||
267 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND | ||
268 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
269 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
270 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | ||
271 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | ||
272 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | ||
273 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||
274 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | ||
275 | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | ||
276 | # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | ||
277 | # SUCH DAMAGE. | ||
278 | # | ||
279 | import threading | ||
280 | import Queue | ||
281 | import itertools | ||
282 | import collections | ||
283 | import time | ||
284 | |||
285 | import multiprocessing | ||
286 | from multiprocessing import Process, cpu_count, TimeoutError, pool | ||
287 | from multiprocessing.util import Finalize, debug | ||
288 | |||
289 | # | ||
290 | # Constants representing the state of a pool | ||
291 | # | ||
292 | |||
293 | RUN = 0 | ||
294 | CLOSE = 1 | ||
295 | TERMINATE = 2 | ||
296 | |||
297 | # | ||
298 | # Miscellaneous | ||
299 | # | ||
300 | |||
301 | def mapstar(args): | ||
302 | return map(*args) | ||
303 | |||
304 | class MaybeEncodingError(Exception): | ||
305 | """Wraps possible unpickleable errors, so they can be | ||
306 | safely sent through the socket.""" | ||
307 | |||
308 | def __init__(self, exc, value): | ||
309 | self.exc = repr(exc) | ||
310 | self.value = repr(value) | ||
311 | super(MaybeEncodingError, self).__init__(self.exc, self.value) | ||
312 | |||
313 | def __str__(self): | ||
314 | return "Error sending result: '%s'. Reason: '%s'" % (self.value, | ||
315 | self.exc) | ||
316 | |||
317 | def __repr__(self): | ||
318 | return "<MaybeEncodingError: %s>" % str(self) | ||
319 | |||
320 | def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): | ||
321 | assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) | ||
322 | put = outqueue.put | ||
323 | get = inqueue.get | ||
324 | if hasattr(inqueue, '_writer'): | ||
325 | inqueue._writer.close() | ||
326 | outqueue._reader.close() | ||
327 | |||
328 | if initializer is not None: | ||
329 | initializer(*initargs) | ||
330 | |||
331 | completed = 0 | ||
332 | while maxtasks is None or (maxtasks and completed < maxtasks): | ||
333 | try: | ||
334 | task = get() | ||
335 | except (EOFError, IOError): | ||
336 | debug('worker got EOFError or IOError -- exiting') | ||
337 | break | ||
338 | |||
339 | if task is None: | ||
340 | debug('worker got sentinel -- exiting') | ||
341 | break | ||
342 | |||
343 | job, i, func, args, kwds = task | ||
344 | try: | ||
345 | result = (True, func(*args, **kwds)) | ||
346 | except Exception as e: | ||
347 | result = (False, e) | ||
348 | try: | ||
349 | put((job, i, result)) | ||
350 | except Exception as e: | ||
351 | wrapped = MaybeEncodingError(e, result[1]) | ||
352 | debug("Possible encoding error while sending result: %s" % ( | ||
353 | wrapped)) | ||
354 | put((job, i, (False, wrapped))) | ||
355 | completed += 1 | ||
356 | debug('worker exiting after %d tasks' % completed) | ||
357 | |||
358 | |||
359 | class Pool(object): | ||
360 | ''' | ||
361 | Class which supports an async version of the `apply()` builtin | ||
362 | ''' | ||
363 | Process = Process | ||
364 | |||
365 | def __init__(self, processes=None, initializer=None, initargs=(), | ||
366 | maxtasksperchild=None): | ||
367 | self._setup_queues() | ||
368 | self._taskqueue = Queue.Queue() | ||
369 | self._cache = {} | ||
370 | self._state = RUN | ||
371 | self._maxtasksperchild = maxtasksperchild | ||
372 | self._initializer = initializer | ||
373 | self._initargs = initargs | ||
374 | |||
375 | if processes is None: | ||
376 | try: | ||
377 | processes = cpu_count() | ||
378 | except NotImplementedError: | ||
379 | processes = 1 | ||
380 | if processes < 1: | ||
381 | raise ValueError("Number of processes must be at least 1") | ||
382 | |||
383 | if initializer is not None and not hasattr(initializer, '__call__'): | ||
384 | raise TypeError('initializer must be a callable') | ||
385 | |||
386 | self._processes = processes | ||
387 | self._pool = [] | ||
388 | self._repopulate_pool() | ||
389 | |||
390 | self._worker_handler = threading.Thread( | ||
391 | target=Pool._handle_workers, | ||
392 | args=(self, ) | ||
393 | ) | ||
394 | self._worker_handler.daemon = True | ||
395 | self._worker_handler._state = RUN | ||
396 | self._worker_handler.start() | ||
397 | |||
398 | |||
399 | self._task_handler = threading.Thread( | ||
400 | target=Pool._handle_tasks, | ||
401 | args=(self._taskqueue, self._quick_put, self._outqueue, self._pool) | ||
402 | ) | ||
403 | self._task_handler.daemon = True | ||
404 | self._task_handler._state = RUN | ||
405 | self._task_handler.start() | ||
406 | |||
407 | self._result_handler = threading.Thread( | ||
408 | target=Pool._handle_results, | ||
409 | args=(self._outqueue, self._quick_get, self._cache) | ||
410 | ) | ||
411 | self._result_handler.daemon = True | ||
412 | self._result_handler._state = RUN | ||
413 | self._result_handler.start() | ||
414 | |||
415 | self._terminate = Finalize( | ||
416 | self, self._terminate_pool, | ||
417 | args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, | ||
418 | self._worker_handler, self._task_handler, | ||
419 | self._result_handler, self._cache), | ||
420 | exitpriority=15 | ||
421 | ) | ||
422 | |||
423 | def _join_exited_workers(self): | ||
424 | """Cleanup after any worker processes which have exited due to reaching | ||
425 | their specified lifetime. Returns True if any workers were cleaned up. | ||
426 | """ | ||
427 | cleaned = False | ||
428 | for i in reversed(range(len(self._pool))): | ||
429 | worker = self._pool[i] | ||
430 | if worker.exitcode is not None: | ||
431 | # worker exited | ||
432 | debug('cleaning up worker %d' % i) | ||
433 | worker.join() | ||
434 | cleaned = True | ||
435 | del self._pool[i] | ||
436 | return cleaned | ||
437 | |||
438 | def _repopulate_pool(self): | ||
439 | """Bring the number of pool processes up to the specified number, | ||
440 | for use after reaping workers which have exited. | ||
441 | """ | ||
442 | for i in range(self._processes - len(self._pool)): | ||
443 | w = self.Process(target=worker, | ||
444 | args=(self._inqueue, self._outqueue, | ||
445 | self._initializer, | ||
446 | self._initargs, self._maxtasksperchild) | ||
447 | ) | ||
448 | self._pool.append(w) | ||
449 | w.name = w.name.replace('Process', 'PoolWorker') | ||
450 | w.daemon = True | ||
451 | w.start() | ||
452 | debug('added worker') | ||
453 | |||
454 | def _maintain_pool(self): | ||
455 | """Clean up any exited workers and start replacements for them. | ||
456 | """ | ||
457 | if self._join_exited_workers(): | ||
458 | self._repopulate_pool() | ||
459 | |||
460 | def _setup_queues(self): | ||
461 | from multiprocessing.queues import SimpleQueue | ||
462 | self._inqueue = SimpleQueue() | ||
463 | self._outqueue = SimpleQueue() | ||
464 | self._quick_put = self._inqueue._writer.send | ||
465 | self._quick_get = self._outqueue._reader.recv | ||
466 | |||
467 | def apply(self, func, args=(), kwds={}): | ||
468 | ''' | ||
469 | Equivalent of `apply()` builtin | ||
470 | ''' | ||
471 | assert self._state == RUN | ||
472 | return self.apply_async(func, args, kwds).get() | ||
473 | |||
474 | def map(self, func, iterable, chunksize=None): | ||
475 | ''' | ||
476 | Equivalent of `map()` builtin | ||
477 | ''' | ||
478 | assert self._state == RUN | ||
479 | return self.map_async(func, iterable, chunksize).get() | ||
480 | |||
481 | def imap(self, func, iterable, chunksize=1): | ||
482 | ''' | ||
483 | Equivalent of `itertools.imap()` -- can be MUCH slower than `Pool.map()` | ||
484 | ''' | ||
485 | assert self._state == RUN | ||
486 | if chunksize == 1: | ||
487 | result = IMapIterator(self._cache) | ||
488 | self._taskqueue.put((((result._job, i, func, (x,), {}) | ||
489 | for i, x in enumerate(iterable)), result._set_length)) | ||
490 | return result | ||
491 | else: | ||
492 | assert chunksize > 1 | ||
493 | task_batches = Pool._get_tasks(func, iterable, chunksize) | ||
494 | result = IMapIterator(self._cache) | ||
495 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) | ||
496 | for i, x in enumerate(task_batches)), result._set_length)) | ||
497 | return (item for chunk in result for item in chunk) | ||
498 | |||
499 | def imap_unordered(self, func, iterable, chunksize=1): | ||
500 | ''' | ||
501 | Like `imap()` method but ordering of results is arbitrary | ||
502 | ''' | ||
503 | assert self._state == RUN | ||
504 | if chunksize == 1: | ||
505 | result = IMapUnorderedIterator(self._cache) | ||
506 | self._taskqueue.put((((result._job, i, func, (x,), {}) | ||
507 | for i, x in enumerate(iterable)), result._set_length)) | ||
508 | return result | ||
509 | else: | ||
510 | assert chunksize > 1 | ||
511 | task_batches = Pool._get_tasks(func, iterable, chunksize) | ||
512 | result = IMapUnorderedIterator(self._cache) | ||
513 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) | ||
514 | for i, x in enumerate(task_batches)), result._set_length)) | ||
515 | return (item for chunk in result for item in chunk) | ||
516 | |||
517 | def apply_async(self, func, args=(), kwds={}, callback=None): | ||
518 | ''' | ||
519 | Asynchronous equivalent of `apply()` builtin | ||
520 | ''' | ||
521 | assert self._state == RUN | ||
522 | result = ApplyResult(self._cache, callback) | ||
523 | self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) | ||
524 | return result | ||
525 | |||
526 | def map_async(self, func, iterable, chunksize=None, callback=None): | ||
527 | ''' | ||
528 | Asynchronous equivalent of `map()` builtin | ||
529 | ''' | ||
530 | assert self._state == RUN | ||
531 | if not hasattr(iterable, '__len__'): | ||
532 | iterable = list(iterable) | ||
533 | |||
534 | if chunksize is None: | ||
535 | chunksize, extra = divmod(len(iterable), len(self._pool) * 4) | ||
536 | if extra: | ||
537 | chunksize += 1 | ||
538 | if len(iterable) == 0: | ||
539 | chunksize = 0 | ||
540 | |||
541 | task_batches = Pool._get_tasks(func, iterable, chunksize) | ||
542 | result = MapResult(self._cache, chunksize, len(iterable), callback) | ||
543 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) | ||
544 | for i, x in enumerate(task_batches)), None)) | ||
545 | return result | ||
546 | |||
547 | @staticmethod | ||
548 | def _handle_workers(pool): | ||
549 | thread = threading.current_thread() | ||
550 | |||
551 | # Keep maintaining workers until the cache gets drained, unless the pool | ||
552 | # is terminated. | ||
553 | while thread._state == RUN or (pool._cache and thread._state != TERMINATE): | ||
554 | pool._maintain_pool() | ||
555 | time.sleep(0.1) | ||
556 | # send sentinel to stop workers | ||
557 | pool._taskqueue.put(None) | ||
558 | debug('worker handler exiting') | ||
559 | |||
560 | @staticmethod | ||
561 | def _handle_tasks(taskqueue, put, outqueue, pool): | ||
562 | thread = threading.current_thread() | ||
563 | |||
564 | for taskseq, set_length in iter(taskqueue.get, None): | ||
565 | i = -1 | ||
566 | for i, task in enumerate(taskseq): | ||
567 | if thread._state: | ||
568 | debug('task handler found thread._state != RUN') | ||
569 | break | ||
570 | try: | ||
571 | put(task) | ||
572 | except IOError: | ||
573 | debug('could not put task on queue') | ||
574 | break | ||
575 | else: | ||
576 | if set_length: | ||
577 | debug('doing set_length()') | ||
578 | set_length(i+1) | ||
579 | continue | ||
580 | break | ||
581 | else: | ||
582 | debug('task handler got sentinel') | ||
583 | |||
584 | |||
585 | try: | ||
586 | # tell result handler to finish when cache is empty | ||
587 | debug('task handler sending sentinel to result handler') | ||
588 | outqueue.put(None) | ||
589 | |||
590 | # tell workers there is no more work | ||
591 | debug('task handler sending sentinel to workers') | ||
592 | for p in pool: | ||
593 | put(None) | ||
594 | except IOError: | ||
595 | debug('task handler got IOError when sending sentinels') | ||
596 | |||
597 | debug('task handler exiting') | ||
598 | |||
599 | @staticmethod | ||
600 | def _handle_results(outqueue, get, cache): | ||
601 | thread = threading.current_thread() | ||
602 | |||
603 | while 1: | ||
604 | try: | ||
605 | task = get() | ||
606 | except (IOError, EOFError): | ||
607 | debug('result handler got EOFError/IOError -- exiting') | ||
608 | return | ||
609 | |||
610 | if thread._state: | ||
611 | assert thread._state == TERMINATE | ||
612 | debug('result handler found thread._state=TERMINATE') | ||
613 | break | ||
614 | |||
615 | if task is None: | ||
616 | debug('result handler got sentinel') | ||
617 | break | ||
618 | |||
619 | job, i, obj = task | ||
620 | try: | ||
621 | cache[job]._set(i, obj) | ||
622 | except KeyError: | ||
623 | pass | ||
624 | |||
625 | while cache and thread._state != TERMINATE: | ||
626 | try: | ||
627 | task = get() | ||
628 | except (IOError, EOFError): | ||
629 | debug('result handler got EOFError/IOError -- exiting') | ||
630 | return | ||
631 | |||
632 | if task is None: | ||
633 | debug('result handler ignoring extra sentinel') | ||
634 | continue | ||
635 | job, i, obj = task | ||
636 | try: | ||
637 | cache[job]._set(i, obj) | ||
638 | except KeyError: | ||
639 | pass | ||
640 | |||
641 | if hasattr(outqueue, '_reader'): | ||
642 | debug('ensuring that outqueue is not full') | ||
643 | # If we don't make room available in outqueue then | ||
644 | # attempts to add the sentinel (None) to outqueue may | ||
645 | # block. There is guaranteed to be no more than 2 sentinels. | ||
646 | try: | ||
647 | for i in range(10): | ||
648 | if not outqueue._reader.poll(): | ||
649 | break | ||
650 | get() | ||
651 | except (IOError, EOFError): | ||
652 | pass | ||
653 | |||
654 | debug('result handler exiting: len(cache)=%s, thread._state=%s', | ||
655 | len(cache), thread._state) | ||
656 | |||
657 | @staticmethod | ||
658 | def _get_tasks(func, it, size): | ||
659 | it = iter(it) | ||
660 | while 1: | ||
661 | x = tuple(itertools.islice(it, size)) | ||
662 | if not x: | ||
663 | return | ||
664 | yield (func, x) | ||
665 | |||
666 | def __reduce__(self): | ||
667 | raise NotImplementedError( | ||
668 | 'pool objects cannot be passed between processes or pickled' | ||
669 | ) | ||
670 | |||
671 | def close(self): | ||
672 | debug('closing pool') | ||
673 | if self._state == RUN: | ||
674 | self._state = CLOSE | ||
675 | self._worker_handler._state = CLOSE | ||
676 | |||
677 | def terminate(self): | ||
678 | debug('terminating pool') | ||
679 | self._state = TERMINATE | ||
680 | self._worker_handler._state = TERMINATE | ||
681 | self._terminate() | ||
682 | |||
683 | def join(self): | ||
684 | debug('joining pool') | ||
685 | assert self._state in (CLOSE, TERMINATE) | ||
686 | self._worker_handler.join() | ||
687 | self._task_handler.join() | ||
688 | self._result_handler.join() | ||
689 | for p in self._pool: | ||
690 | p.join() | ||
691 | |||
692 | @staticmethod | ||
693 | def _help_stuff_finish(inqueue, task_handler, size): | ||
694 | # task_handler may be blocked trying to put items on inqueue | ||
695 | debug('removing tasks from inqueue until task handler finished') | ||
696 | inqueue._rlock.acquire() | ||
697 | while task_handler.is_alive() and inqueue._reader.poll(): | ||
698 | inqueue._reader.recv() | ||
699 | time.sleep(0) | ||
700 | |||
701 | @classmethod | ||
702 | def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, | ||
703 | worker_handler, task_handler, result_handler, cache): | ||
704 | # this is guaranteed to only be called once | ||
705 | debug('finalizing pool') | ||
706 | |||
707 | worker_handler._state = TERMINATE | ||
708 | task_handler._state = TERMINATE | ||
709 | |||
710 | debug('helping task handler/workers to finish') | ||
711 | cls._help_stuff_finish(inqueue, task_handler, len(pool)) | ||
712 | |||
713 | assert result_handler.is_alive() or len(cache) == 0 | ||
714 | |||
715 | result_handler._state = TERMINATE | ||
716 | outqueue.put(None) # sentinel | ||
717 | |||
718 | # We must wait for the worker handler to exit before terminating | ||
719 | # workers because we don't want workers to be restarted behind our back. | ||
720 | debug('joining worker handler') | ||
721 | if threading.current_thread() is not worker_handler: | ||
722 | worker_handler.join(1e100) | ||
723 | |||
724 | # Terminate workers which haven't already finished. | ||
725 | if pool and hasattr(pool[0], 'terminate'): | ||
726 | debug('terminating workers') | ||
727 | for p in pool: | ||
728 | if p.exitcode is None: | ||
729 | p.terminate() | ||
730 | |||
731 | debug('joining task handler') | ||
732 | if threading.current_thread() is not task_handler: | ||
733 | task_handler.join(1e100) | ||
734 | |||
735 | debug('joining result handler') | ||
736 | if threading.current_thread() is not result_handler: | ||
737 | result_handler.join(1e100) | ||
738 | |||
739 | if pool and hasattr(pool[0], 'terminate'): | ||
740 | debug('joining pool workers') | ||
741 | for p in pool: | ||
742 | if p.is_alive(): | ||
743 | # worker has not yet exited | ||
744 | debug('cleaning up worker %d' % p.pid) | ||
745 | p.join() | ||
746 | |||
747 | class ApplyResult(object): | ||
748 | |||
749 | def __init__(self, cache, callback): | ||
750 | self._cond = threading.Condition(threading.Lock()) | ||
751 | self._job = multiprocessing.pool.job_counter.next() | ||
752 | self._cache = cache | ||
753 | self._ready = False | ||
754 | self._callback = callback | ||
755 | cache[self._job] = self | ||
756 | |||
757 | def ready(self): | ||
758 | return self._ready | ||
759 | |||
760 | def successful(self): | ||
761 | assert self._ready | ||
762 | return self._success | ||
763 | |||
764 | def wait(self, timeout=None): | ||
765 | self._cond.acquire() | ||
766 | try: | ||
767 | if not self._ready: | ||
768 | self._cond.wait(timeout) | ||
769 | finally: | ||
770 | self._cond.release() | ||
771 | |||
772 | def get(self, timeout=None): | ||
773 | self.wait(timeout) | ||
774 | if not self._ready: | ||
775 | raise TimeoutError | ||
776 | if self._success: | ||
777 | return self._value | ||
778 | else: | ||
779 | raise self._value | ||
780 | |||
781 | def _set(self, i, obj): | ||
782 | self._success, self._value = obj | ||
783 | if self._callback and self._success: | ||
784 | self._callback(self._value) | ||
785 | self._cond.acquire() | ||
786 | try: | ||
787 | self._ready = True | ||
788 | self._cond.notify() | ||
789 | finally: | ||
790 | self._cond.release() | ||
791 | del self._cache[self._job] | ||
792 | |||
793 | # | ||
794 | # Class whose instances are returned by `Pool.map_async()` | ||
795 | # | ||
796 | |||
797 | class MapResult(ApplyResult): | ||
798 | |||
799 | def __init__(self, cache, chunksize, length, callback): | ||
800 | ApplyResult.__init__(self, cache, callback) | ||
801 | self._success = True | ||
802 | self._value = [None] * length | ||
803 | self._chunksize = chunksize | ||
804 | if chunksize <= 0: | ||
805 | self._number_left = 0 | ||
806 | self._ready = True | ||
807 | del cache[self._job] | ||
808 | else: | ||
809 | self._number_left = length//chunksize + bool(length % chunksize) | ||
810 | |||
811 | def _set(self, i, success_result): | ||
812 | success, result = success_result | ||
813 | if success: | ||
814 | self._value[i*self._chunksize:(i+1)*self._chunksize] = result | ||
815 | self._number_left -= 1 | ||
816 | if self._number_left == 0: | ||
817 | if self._callback: | ||
818 | self._callback(self._value) | ||
819 | del self._cache[self._job] | ||
820 | self._cond.acquire() | ||
821 | try: | ||
822 | self._ready = True | ||
823 | self._cond.notify() | ||
824 | finally: | ||
825 | self._cond.release() | ||
826 | |||
827 | else: | ||
828 | self._success = False | ||
829 | self._value = result | ||
830 | del self._cache[self._job] | ||
831 | self._cond.acquire() | ||
832 | try: | ||
833 | self._ready = True | ||
834 | self._cond.notify() | ||
835 | finally: | ||
836 | self._cond.release() | ||
837 | |||
838 | # | ||
839 | # Class whose instances are returned by `Pool.imap()` | ||
840 | # | ||
841 | |||
842 | class IMapIterator(object): | ||
843 | |||
844 | def __init__(self, cache): | ||
845 | self._cond = threading.Condition(threading.Lock()) | ||
846 | self._job = multiprocessing.pool.job_counter.next() | ||
847 | self._cache = cache | ||
848 | self._items = collections.deque() | ||
849 | self._index = 0 | ||
850 | self._length = None | ||
851 | self._unsorted = {} | ||
852 | cache[self._job] = self | ||
853 | |||
854 | def __iter__(self): | ||
855 | return self | ||
856 | |||
857 | def next(self, timeout=None): | ||
858 | self._cond.acquire() | ||
859 | try: | ||
860 | try: | ||
861 | item = self._items.popleft() | ||
862 | except IndexError: | ||
863 | if self._index == self._length: | ||
864 | raise StopIteration | ||
865 | self._cond.wait(timeout) | ||
866 | try: | ||
867 | item = self._items.popleft() | ||
868 | except IndexError: | ||
869 | if self._index == self._length: | ||
870 | raise StopIteration | ||
871 | raise TimeoutError | ||
872 | finally: | ||
873 | self._cond.release() | ||
874 | |||
875 | success, value = item | ||
876 | if success: | ||
877 | return value | ||
878 | raise value | ||
879 | |||
880 | __next__ = next # XXX | ||
881 | |||
882 | def _set(self, i, obj): | ||
883 | self._cond.acquire() | ||
884 | try: | ||
885 | if self._index == i: | ||
886 | self._items.append(obj) | ||
887 | self._index += 1 | ||
888 | while self._index in self._unsorted: | ||
889 | obj = self._unsorted.pop(self._index) | ||
890 | self._items.append(obj) | ||
891 | self._index += 1 | ||
892 | self._cond.notify() | ||
893 | else: | ||
894 | self._unsorted[i] = obj | ||
895 | |||
896 | if self._index == self._length: | ||
897 | del self._cache[self._job] | ||
898 | finally: | ||
899 | self._cond.release() | ||
900 | |||
901 | def _set_length(self, length): | ||
902 | self._cond.acquire() | ||
903 | try: | ||
904 | self._length = length | ||
905 | if self._index == self._length: | ||
906 | self._cond.notify() | ||
907 | del self._cache[self._job] | ||
908 | finally: | ||
909 | self._cond.release() | ||
910 | |||
911 | # | ||
912 | # Class whose instances are returned by `Pool.imap_unordered()` | ||
913 | # | ||
914 | |||
915 | class IMapUnorderedIterator(IMapIterator): | ||
916 | |||
917 | def _set(self, i, obj): | ||
918 | self._cond.acquire() | ||
919 | try: | ||
920 | self._items.append(obj) | ||
921 | self._index += 1 | ||
922 | self._cond.notify() | ||
923 | if self._index == self._length: | ||
924 | del self._cache[self._job] | ||
925 | finally: | ||
926 | self._cond.release() | ||
927 | 5 | ||
928 | 6 | ||
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index d73fe827e4..0d4a26ced0 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py | |||
@@ -142,52 +142,6 @@ class ProcessServer(Process, BaseImplServer): | |||
142 | def stop(self): | 142 | def stop(self): |
143 | self.keep_running.clear() | 143 | self.keep_running.clear() |
144 | 144 | ||
145 | def bootstrap_2_6_6(self): | ||
146 | """Pulled from python 2.6.6. Needed to ensure we have the fix from | ||
147 | http://bugs.python.org/issue5313 when running on python version 2.6.2 | ||
148 | or lower.""" | ||
149 | |||
150 | try: | ||
151 | self._children = set() | ||
152 | self._counter = itertools.count(1) | ||
153 | try: | ||
154 | sys.stdin.close() | ||
155 | sys.stdin = open(os.devnull) | ||
156 | except (OSError, ValueError): | ||
157 | pass | ||
158 | multiprocessing._current_process = self | ||
159 | util._finalizer_registry.clear() | ||
160 | util._run_after_forkers() | ||
161 | util.info('child process calling self.run()') | ||
162 | try: | ||
163 | self.run() | ||
164 | exitcode = 0 | ||
165 | finally: | ||
166 | util._exit_function() | ||
167 | except SystemExit as e: | ||
168 | if not e.args: | ||
169 | exitcode = 1 | ||
170 | elif type(e.args[0]) is int: | ||
171 | exitcode = e.args[0] | ||
172 | else: | ||
173 | sys.stderr.write(e.args[0] + '\n') | ||
174 | sys.stderr.flush() | ||
175 | exitcode = 1 | ||
176 | except: | ||
177 | exitcode = 1 | ||
178 | import traceback | ||
179 | sys.stderr.write('Process %s:\n' % self.name) | ||
180 | sys.stderr.flush() | ||
181 | traceback.print_exc() | ||
182 | |||
183 | util.info('process exiting with exitcode %d' % exitcode) | ||
184 | return exitcode | ||
185 | |||
186 | # Python versions 2.6.0 through 2.6.2 suffer from a multiprocessing bug | ||
187 | # which can result in a bitbake server hang during the parsing process | ||
188 | if (2, 6, 0) <= sys.version_info < (2, 6, 3): | ||
189 | _bootstrap = bootstrap_2_6_6 | ||
190 | |||
191 | class BitBakeProcessServerConnection(BitBakeBaseServerConnection): | 145 | class BitBakeProcessServerConnection(BitBakeBaseServerConnection): |
192 | def __init__(self, serverImpl, ui_channel, event_queue): | 146 | def __init__(self, serverImpl, ui_channel, event_queue): |
193 | self.procserver = serverImpl | 147 | self.procserver = serverImpl |
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py index 359d5adb67..5045e55ae2 100644 --- a/bitbake/lib/bb/server/xmlrpc.py +++ b/bitbake/lib/bb/server/xmlrpc.py | |||
@@ -51,100 +51,18 @@ import inspect, select | |||
51 | 51 | ||
52 | from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer | 52 | from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer |
53 | 53 | ||
54 | if sys.hexversion < 0x020600F0: | 54 | class BBTransport(xmlrpclib.Transport): |
55 | print("Sorry, python 2.6 or later is required for bitbake's XMLRPC mode") | 55 | def __init__(self): |
56 | sys.exit(1) | 56 | self.connection_token = None |
57 | 57 | xmlrpclib.Transport.__init__(self) | |
58 | ## | ||
59 | # The xmlrpclib.Transport class has undergone various changes in Python 2.7 | ||
60 | # which break BitBake's XMLRPC implementation. | ||
61 | # To work around this we subclass Transport and have a copy/paste of method | ||
62 | # implementations from Python 2.6.6's xmlrpclib. | ||
63 | # | ||
64 | # Upstream Python bug is #8194 (http://bugs.python.org/issue8194) | ||
65 | # This bug is relevant for Python 2.7.0 and 2.7.1 but was fixed for | ||
66 | # Python > 2.7.2 | ||
67 | # | ||
68 | # To implement a simple form of client control, we use a special transport | ||
69 | # that adds a HTTP header field ("Bitbake-token") to ensure that a server | ||
70 | # can communicate with only a client at a given time (the client must use | ||
71 | # the same token). | ||
72 | ## | ||
73 | if (2, 7, 0) <= sys.version_info < (2, 7, 2): | ||
74 | class BBTransport(xmlrpclib.Transport): | ||
75 | def __init__(self): | ||
76 | self.connection_token = None | ||
77 | xmlrpclib.Transport.__init__(self) | ||
78 | |||
79 | def request(self, host, handler, request_body, verbose=0): | ||
80 | h = self.make_connection(host) | ||
81 | if verbose: | ||
82 | h.set_debuglevel(1) | ||
83 | |||
84 | self.send_request(h, handler, request_body) | ||
85 | self.send_host(h, host) | ||
86 | self.send_user_agent(h) | ||
87 | if self.connection_token: | ||
88 | h.putheader("Bitbake-token", self.connection_token) | ||
89 | self.send_content(h, request_body) | ||
90 | |||
91 | errcode, errmsg, headers = h.getreply() | ||
92 | |||
93 | if errcode != 200: | ||
94 | raise ProtocolError( | ||
95 | host + handler, | ||
96 | errcode, errmsg, | ||
97 | headers | ||
98 | ) | ||
99 | |||
100 | self.verbose = verbose | ||
101 | 58 | ||
102 | try: | 59 | def set_connection_token(self, token): |
103 | sock = h._conn.sock | 60 | self.connection_token = token |
104 | except AttributeError: | 61 | |
105 | sock = None | 62 | def send_content(self, h, body): |
106 | 63 | if self.connection_token: | |
107 | return self._parse_response(h.getfile(), sock) | 64 | h.putheader("Bitbake-token", self.connection_token) |
108 | 65 | xmlrpclib.Transport.send_content(self, h, body) | |
109 | def make_connection(self, host): | ||
110 | import httplib | ||
111 | host, extra_headers, x509 = self.get_host_info(host) | ||
112 | return httplib.HTTP(host) | ||
113 | |||
114 | def _parse_response(self, file, sock): | ||
115 | p, u = self.getparser() | ||
116 | |||
117 | while 1: | ||
118 | if sock: | ||
119 | response = sock.recv(1024) | ||
120 | else: | ||
121 | response = file.read(1024) | ||
122 | if not response: | ||
123 | break | ||
124 | if self.verbose: | ||
125 | print("body:", repr(response)) | ||
126 | p.feed(response) | ||
127 | |||
128 | file.close() | ||
129 | p.close() | ||
130 | |||
131 | return u.close() | ||
132 | |||
133 | def set_connection_token(self, token): | ||
134 | self.connection_token = token | ||
135 | else: | ||
136 | class BBTransport(xmlrpclib.Transport): | ||
137 | def __init__(self): | ||
138 | self.connection_token = None | ||
139 | xmlrpclib.Transport.__init__(self) | ||
140 | |||
141 | def set_connection_token(self, token): | ||
142 | self.connection_token = token | ||
143 | |||
144 | def send_content(self, h, body): | ||
145 | if self.connection_token: | ||
146 | h.putheader("Bitbake-token", self.connection_token) | ||
147 | xmlrpclib.Transport.send_content(self, h, body) | ||
148 | 66 | ||
149 | def _create_server(host, port): | 67 | def _create_server(host, port): |
150 | t = BBTransport() | 68 | t = BBTransport() |
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 7db6e3862f..5301ebaa97 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py | |||
@@ -860,11 +860,8 @@ def process_profilelog(fn): | |||
860 | pout.close() | 860 | pout.close() |
861 | 861 | ||
862 | # | 862 | # |
863 | # Work around multiprocessing pool bugs in python < 2.7.3 | 863 | # Was present to work around multiprocessing pool bugs in python < 2.7.3 |
864 | # | 864 | # |
865 | def multiprocessingpool(*args, **kwargs): | 865 | def multiprocessingpool(*args, **kwargs): |
866 | if sys.version_info < (2, 7, 3): | 866 | return multiprocessing.pool.Pool(*args, **kwargs) |
867 | return bb.compat.Pool(*args, **kwargs) | ||
868 | else: | ||
869 | return multiprocessing.pool.Pool(*args, **kwargs) | ||
870 | 867 | ||