diff options
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/lib/bb/cache.py | 177 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 16 |
2 files changed, 127 insertions, 66 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 0620621d0b..5b8e3ee8f4 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -43,8 +43,10 @@ except ImportError: | |||
43 | logger.info("Importing cPickle failed. " | 43 | logger.info("Importing cPickle failed. " |
44 | "Falling back to a very slow implementation.") | 44 | "Falling back to a very slow implementation.") |
45 | 45 | ||
46 | __cache_version__ = "138" | 46 | __cache_version__ = "139" |
47 | 47 | ||
48 | def getCacheFile(path, filename): | ||
49 | return os.path.join(path, filename) | ||
48 | 50 | ||
49 | # RecipeInfoCommon defines common data retrieving methods | 51 | # RecipeInfoCommon defines common data retrieving methods |
50 | # from meta data for caches. CoreRecipeInfo as well as other | 52 | # from meta data for caches. CoreRecipeInfo as well as other |
@@ -86,12 +88,9 @@ class RecipeInfoCommon(object): | |||
86 | class CoreRecipeInfo(RecipeInfoCommon): | 88 | class CoreRecipeInfo(RecipeInfoCommon): |
87 | __slots__ = () | 89 | __slots__ = () |
88 | 90 | ||
89 | def __init__(self, filename, metadata): | 91 | cachefile = "bb_cache.dat" |
90 | self.name = "base" | ||
91 | # please override this member with the correct data cache file | ||
92 | # such as (bb_cache.dat, bb_extracache_hob.dat) | ||
93 | self.cachefile = "bb_cache.dat" | ||
94 | 92 | ||
93 | def __init__(self, filename, metadata): | ||
95 | self.file_depends = metadata.getVar('__depends', False) | 94 | self.file_depends = metadata.getVar('__depends', False) |
96 | self.timestamp = bb.parse.cached_mtime(filename) | 95 | self.timestamp = bb.parse.cached_mtime(filename) |
97 | self.variants = self.listvar('__VARIANTS', metadata) + [''] | 96 | self.variants = self.listvar('__VARIANTS', metadata) + [''] |
@@ -265,7 +264,7 @@ class Cache(object): | |||
265 | return | 264 | return |
266 | 265 | ||
267 | self.has_cache = True | 266 | self.has_cache = True |
268 | self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") | 267 | self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat") |
269 | 268 | ||
270 | logger.debug(1, "Using cache in '%s'", self.cachedir) | 269 | logger.debug(1, "Using cache in '%s'", self.cachedir) |
271 | bb.utils.mkdirhier(self.cachedir) | 270 | bb.utils.mkdirhier(self.cachedir) |
@@ -279,12 +278,21 @@ class Cache(object): | |||
279 | old_mtimes.append(newest_mtime) | 278 | old_mtimes.append(newest_mtime) |
280 | newest_mtime = max(old_mtimes) | 279 | newest_mtime = max(old_mtimes) |
281 | 280 | ||
282 | if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: | 281 | bNeedUpdate = True |
282 | if self.caches_array: | ||
283 | for cache_class in self.caches_array: | ||
284 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
285 | cachefile = getCacheFile(self.cachedir, cache_class.cachefile) | ||
286 | bNeedUpdate = bNeedUpdate and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime) | ||
287 | cache_class.init_cacheData(self) | ||
288 | if bNeedUpdate: | ||
283 | self.load_cachefile() | 289 | self.load_cachefile() |
284 | elif os.path.isfile(self.cachefile): | 290 | elif os.path.isfile(self.cachefile): |
285 | logger.info("Out of date cache found, rebuilding...") | 291 | logger.info("Out of date cache found, rebuilding...") |
286 | 292 | ||
287 | def load_cachefile(self): | 293 | def load_cachefile(self): |
294 | # Firstly, using core cache file information for | ||
295 | # valid checking | ||
288 | with open(self.cachefile, "rb") as cachefile: | 296 | with open(self.cachefile, "rb") as cachefile: |
289 | pickled = pickle.Unpickler(cachefile) | 297 | pickled = pickle.Unpickler(cachefile) |
290 | try: | 298 | try: |
@@ -301,31 +309,52 @@ class Cache(object): | |||
301 | logger.info('Bitbake version mismatch, rebuilding...') | 309 | logger.info('Bitbake version mismatch, rebuilding...') |
302 | return | 310 | return |
303 | 311 | ||
304 | cachesize = os.fstat(cachefile.fileno()).st_size | ||
305 | bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) | ||
306 | 312 | ||
307 | previous_percent = 0 | 313 | cachesize = 0 |
308 | while cachefile: | 314 | previous_progress = 0 |
309 | try: | 315 | previous_percent = 0 |
310 | key = pickled.load() | ||
311 | value = pickled.load() | ||
312 | except Exception: | ||
313 | break | ||
314 | 316 | ||
315 | self.depends_cache[key] = value | 317 | # Calculate the correct cachesize of all those cache files |
316 | 318 | for cache_class in self.caches_array: | |
317 | # only fire events on even percentage boundaries | 319 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): |
318 | current_progress = cachefile.tell() | 320 | cachefile = getCacheFile(self.cachedir, cache_class.cachefile) |
319 | current_percent = 100 * current_progress / cachesize | 321 | with open(cachefile, "rb") as cachefile: |
320 | if current_percent > previous_percent: | 322 | cachesize += os.fstat(cachefile.fileno()).st_size |
321 | previous_percent = current_percent | ||
322 | bb.event.fire(bb.event.CacheLoadProgress(current_progress), | ||
323 | self.data) | ||
324 | |||
325 | bb.event.fire(bb.event.CacheLoadCompleted(cachesize, | ||
326 | len(self.depends_cache)), | ||
327 | self.data) | ||
328 | 323 | ||
324 | bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) | ||
325 | |||
326 | for cache_class in self.caches_array: | ||
327 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
328 | cachefile = getCacheFile(self.cachedir, cache_class.cachefile) | ||
329 | with open(cachefile, "rb") as cachefile: | ||
330 | pickled = pickle.Unpickler(cachefile) | ||
331 | while cachefile: | ||
332 | try: | ||
333 | key = pickled.load() | ||
334 | value = pickled.load() | ||
335 | except Exception: | ||
336 | break | ||
337 | if self.depends_cache.has_key(key): | ||
338 | self.depends_cache[key].append(value) | ||
339 | else: | ||
340 | self.depends_cache[key] = [value] | ||
341 | # only fire events on even percentage boundaries | ||
342 | current_progress = cachefile.tell() + previous_progress | ||
343 | current_percent = 100 * current_progress / cachesize | ||
344 | if current_percent > previous_percent: | ||
345 | previous_percent = current_percent | ||
346 | bb.event.fire(bb.event.CacheLoadProgress(current_progress), | ||
347 | self.data) | ||
348 | |||
349 | previous_progress += current_progress | ||
350 | |||
351 | # Note: depends cache number is corresponding to the parsing file numbers. | ||
352 | # The same file has several caches, still regarded as one item in the cache | ||
353 | bb.event.fire(bb.event.CacheLoadCompleted(cachesize, | ||
354 | len(self.depends_cache)), | ||
355 | self.data) | ||
356 | |||
357 | |||
329 | @staticmethod | 358 | @staticmethod |
330 | def virtualfn2realfn(virtualfn): | 359 | def virtualfn2realfn(virtualfn): |
331 | """ | 360 | """ |
@@ -376,8 +405,14 @@ class Cache(object): | |||
376 | depends |= (data.getVar("__depends", False) or set()) | 405 | depends |= (data.getVar("__depends", False) or set()) |
377 | if depends and not variant: | 406 | if depends and not variant: |
378 | data.setVar("__depends", depends) | 407 | data.setVar("__depends", depends) |
379 | info = CoreRecipeInfo(filename, data) | 408 | |
380 | infos.append((virtualfn, info)) | 409 | info_array = [] |
410 | for cache_class in caches_array: | ||
411 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
412 | info = cache_class(filename, data) | ||
413 | info_array.append(info) | ||
414 | infos.append((virtualfn, info_array)) | ||
415 | |||
381 | return infos | 416 | return infos |
382 | 417 | ||
383 | def load(self, filename, appends, configdata): | 418 | def load(self, filename, appends, configdata): |
@@ -391,8 +426,9 @@ class Cache(object): | |||
391 | cached = self.cacheValid(filename) | 426 | cached = self.cacheValid(filename) |
392 | if cached: | 427 | if cached: |
393 | infos = [] | 428 | infos = [] |
394 | info = self.depends_cache[filename] | 429 | # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] |
395 | for variant in info.variants: | 430 | info_array = self.depends_cache[filename] |
431 | for variant in info_array[0].variants: | ||
396 | virtualfn = self.realfn2virtual(filename, variant) | 432 | virtualfn = self.realfn2virtual(filename, variant) |
397 | infos.append((virtualfn, self.depends_cache[virtualfn])) | 433 | infos.append((virtualfn, self.depends_cache[virtualfn])) |
398 | else: | 434 | else: |
@@ -408,12 +444,12 @@ class Cache(object): | |||
408 | skipped, virtuals = 0, 0 | 444 | skipped, virtuals = 0, 0 |
409 | 445 | ||
410 | cached, infos = self.load(fn, appends, cfgData) | 446 | cached, infos = self.load(fn, appends, cfgData) |
411 | for virtualfn, info in infos: | 447 | for virtualfn, info_array in infos: |
412 | if info.skipped: | 448 | if info_array[0].skipped: |
413 | logger.debug(1, "Skipping %s", virtualfn) | 449 | logger.debug(1, "Skipping %s", virtualfn) |
414 | skipped += 1 | 450 | skipped += 1 |
415 | else: | 451 | else: |
416 | self.add_info(virtualfn, info, cacheData, not cached) | 452 | self.add_info(virtualfn, info_array, cacheData, not cached) |
417 | virtuals += 1 | 453 | virtuals += 1 |
418 | 454 | ||
419 | return cached, skipped, virtuals | 455 | return cached, skipped, virtuals |
@@ -457,15 +493,15 @@ class Cache(object): | |||
457 | self.remove(fn) | 493 | self.remove(fn) |
458 | return False | 494 | return False |
459 | 495 | ||
460 | info = self.depends_cache[fn] | 496 | info_array = self.depends_cache[fn] |
461 | # Check the file's timestamp | 497 | # Check the file's timestamp |
462 | if mtime != info.timestamp: | 498 | if mtime != info_array[0].timestamp: |
463 | logger.debug(2, "Cache: %s changed", fn) | 499 | logger.debug(2, "Cache: %s changed", fn) |
464 | self.remove(fn) | 500 | self.remove(fn) |
465 | return False | 501 | return False |
466 | 502 | ||
467 | # Check dependencies are still valid | 503 | # Check dependencies are still valid |
468 | depends = info.file_depends | 504 | depends = info_array[0].file_depends |
469 | if depends: | 505 | if depends: |
470 | for f, old_mtime in depends: | 506 | for f, old_mtime in depends: |
471 | fmtime = bb.parse.cached_mtime_noerror(f) | 507 | fmtime = bb.parse.cached_mtime_noerror(f) |
@@ -483,7 +519,7 @@ class Cache(object): | |||
483 | return False | 519 | return False |
484 | 520 | ||
485 | invalid = False | 521 | invalid = False |
486 | for cls in info.variants: | 522 | for cls in info_array[0].variants: |
487 | virtualfn = self.realfn2virtual(fn, cls) | 523 | virtualfn = self.realfn2virtual(fn, cls) |
488 | self.clean.add(virtualfn) | 524 | self.clean.add(virtualfn) |
489 | if virtualfn not in self.depends_cache: | 525 | if virtualfn not in self.depends_cache: |
@@ -530,13 +566,30 @@ class Cache(object): | |||
530 | logger.debug(2, "Cache is clean, not saving.") | 566 | logger.debug(2, "Cache is clean, not saving.") |
531 | return | 567 | return |
532 | 568 | ||
533 | with open(self.cachefile, "wb") as cachefile: | 569 | file_dict = {} |
534 | pickler = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL) | 570 | pickler_dict = {} |
535 | pickler.dump(__cache_version__) | 571 | for cache_class in self.caches_array: |
536 | pickler.dump(bb.__version__) | 572 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): |
537 | for key, value in self.depends_cache.iteritems(): | 573 | cache_class_name = cache_class.__name__ |
538 | pickler.dump(key) | 574 | cachefile = getCacheFile(self.cachedir, cache_class.cachefile) |
539 | pickler.dump(value) | 575 | file_dict[cache_class_name] = open(cachefile, "wb") |
576 | pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL) | ||
577 | |||
578 | pickler_dict['CoreRecipeInfo'].dump(__cache_version__) | ||
579 | pickler_dict['CoreRecipeInfo'].dump(bb.__version__) | ||
580 | |||
581 | try: | ||
582 | for key, info_array in self.depends_cache.iteritems(): | ||
583 | for info in info_array: | ||
584 | if isinstance(info, RecipeInfoCommon): | ||
585 | cache_class_name = info.__class__.__name__ | ||
586 | pickler_dict[cache_class_name].dump(key) | ||
587 | pickler_dict[cache_class_name].dump(info) | ||
588 | finally: | ||
589 | for cache_class in self.caches_array: | ||
590 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
591 | cache_class_name = cache_class.__name__ | ||
592 | file_dict[cache_class_name].close() | ||
540 | 593 | ||
541 | del self.depends_cache | 594 | del self.depends_cache |
542 | 595 | ||
@@ -544,17 +597,17 @@ class Cache(object): | |||
544 | def mtime(cachefile): | 597 | def mtime(cachefile): |
545 | return bb.parse.cached_mtime_noerror(cachefile) | 598 | return bb.parse.cached_mtime_noerror(cachefile) |
546 | 599 | ||
547 | def add_info(self, filename, info, cacheData, parsed=None): | 600 | def add_info(self, filename, info_array, cacheData, parsed=None): |
548 | if not info.skipped: | 601 | if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): |
549 | cacheData.add_from_recipeinfo(filename, info) | 602 | cacheData.add_from_recipeinfo(filename, info_array) |
550 | 603 | ||
551 | if not self.has_cache: | 604 | if not self.has_cache: |
552 | return | 605 | return |
553 | 606 | ||
554 | if (info.skipped or 'SRCREVINACTION' not in info.pv) and not info.nocache: | 607 | if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: |
555 | if parsed: | 608 | if parsed: |
556 | self.cacheclean = False | 609 | self.cacheclean = False |
557 | self.depends_cache[filename] = info | 610 | self.depends_cache[filename] = info_array |
558 | 611 | ||
559 | def add(self, file_name, data, cacheData, parsed=None): | 612 | def add(self, file_name, data, cacheData, parsed=None): |
560 | """ | 613 | """ |
@@ -562,8 +615,12 @@ class Cache(object): | |||
562 | """ | 615 | """ |
563 | 616 | ||
564 | realfn = self.virtualfn2realfn(file_name)[0] | 617 | realfn = self.virtualfn2realfn(file_name)[0] |
565 | info = CoreRecipeInfo(realfn, data) | 618 | |
566 | self.add_info(file_name, info, cacheData, parsed) | 619 | info_array = [] |
620 | for cache_class in self.caches_array: | ||
621 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
622 | info_array.append(cache_class(realfn, data)) | ||
623 | self.add_info(file_name, info_array, cacheData, parsed) | ||
567 | 624 | ||
568 | @staticmethod | 625 | @staticmethod |
569 | def load_bbfile(bbfile, appends, config): | 626 | def load_bbfile(bbfile, appends, config): |
@@ -629,7 +686,10 @@ class CacheData(object): | |||
629 | 686 | ||
630 | def __init__(self, caches_array): | 687 | def __init__(self, caches_array): |
631 | self.caches_array = caches_array | 688 | self.caches_array = caches_array |
632 | CoreRecipeInfo.init_cacheData(self) | 689 | for cache_class in self.caches_array: |
690 | if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): | ||
691 | cache_class.init_cacheData(self) | ||
692 | |||
633 | # Direct cache variables | 693 | # Direct cache variables |
634 | self.task_queues = {} | 694 | self.task_queues = {} |
635 | self.preferred = {} | 695 | self.preferred = {} |
@@ -640,7 +700,8 @@ class CacheData(object): | |||
640 | self.bbfile_priority = {} | 700 | self.bbfile_priority = {} |
641 | self.bbfile_config_priorities = [] | 701 | self.bbfile_config_priorities = [] |
642 | 702 | ||
643 | def add_from_recipeinfo(self, fn, info): | 703 | def add_from_recipeinfo(self, fn, info_array): |
644 | info.add_cacheData(self, fn) | 704 | for info in info_array: |
705 | info.add_cacheData(self, fn) | ||
645 | 706 | ||
646 | 707 | ||
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 89da7e99cf..8379d0caf1 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -773,13 +773,13 @@ class BBCooker: | |||
773 | 773 | ||
774 | fn = bb.cache.Cache.realfn2virtual(fn, cls) | 774 | fn = bb.cache.Cache.realfn2virtual(fn, cls) |
775 | try: | 775 | try: |
776 | maininfo = infos[fn] | 776 | info_array = infos[fn] |
777 | except KeyError: | 777 | except KeyError: |
778 | bb.fatal("%s does not exist" % fn) | 778 | bb.fatal("%s does not exist" % fn) |
779 | self.status.add_from_recipeinfo(fn, maininfo) | 779 | self.status.add_from_recipeinfo(fn, info_array) |
780 | 780 | ||
781 | # Tweak some variables | 781 | # Tweak some variables |
782 | item = maininfo.pn | 782 | item = info_array[0].pn |
783 | self.status.ignored_dependencies = set() | 783 | self.status.ignored_dependencies = set() |
784 | self.status.bbfile_priority[fn] = 1 | 784 | self.status.bbfile_priority[fn] = 1 |
785 | 785 | ||
@@ -1231,10 +1231,10 @@ class CookerParser(object): | |||
1231 | else: | 1231 | else: |
1232 | self.cached += 1 | 1232 | self.cached += 1 |
1233 | 1233 | ||
1234 | for virtualfn, info in result: | 1234 | for virtualfn, info_array in result: |
1235 | if info.skipped: | 1235 | if info_array[0].skipped: |
1236 | self.skipped += 1 | 1236 | self.skipped += 1 |
1237 | self.bb_cache.add_info(virtualfn, info, self.cooker.status, | 1237 | self.bb_cache.add_info(virtualfn, info_array, self.cooker.status, |
1238 | parsed=parsed) | 1238 | parsed=parsed) |
1239 | return True | 1239 | return True |
1240 | 1240 | ||
@@ -1242,5 +1242,5 @@ class CookerParser(object): | |||
1242 | infos = self.bb_cache.parse(filename, | 1242 | infos = self.bb_cache.parse(filename, |
1243 | self.cooker.get_file_appends(filename), | 1243 | self.cooker.get_file_appends(filename), |
1244 | self.cfgdata, self.cooker.caches_array) | 1244 | self.cfgdata, self.cooker.caches_array) |
1245 | for vfn, info in infos: | 1245 | for vfn, info_array in infos: |
1246 | self.cooker.status.add_from_recipeinfo(vfn, info) | 1246 | self.cooker.status.add_from_recipeinfo(vfn, info_array) |