From 5af197b55a4b779f1ec93186f0723026949ba2b5 Mon Sep 17 00:00:00 2001 From: Liping Ke Date: Fri, 3 Jun 2011 08:22:40 +0800 Subject: cache: Implement multiple extra cache fields request support This patch is to support extra cache. If user needs to request extra cache fields besides CoreRecipeInfo fields, just add a new XXXRecipeInfo class definition as Hob Does. Currently supported Extra RecipeInfo name is an array. We can support multiple extra cache fields at the same time besides CoreRecipeInfo which is needed by all clients. (Bitbake rev: 077657e50ad032c0fa876bf54e9802af2686e0fb) Signed-off-by: Liping Ke Signed-off-by: Richard Purdie --- bitbake/lib/bb/cache.py | 177 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 119 insertions(+), 58 deletions(-) (limited to 'bitbake/lib/bb/cache.py') diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 0620621d0b..5b8e3ee8f4 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -43,8 +43,10 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "138" +__cache_version__ = "139" +def getCacheFile(path, filename): + return os.path.join(path, filename) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other @@ -86,12 +88,9 @@ class RecipeInfoCommon(object): class CoreRecipeInfo(RecipeInfoCommon): __slots__ = () - def __init__(self, filename, metadata): - self.name = "base" - # please override this member with the correct data cache file - # such as (bb_cache.dat, bb_extracache_hob.dat) - self.cachefile = "bb_cache.dat" + cachefile = "bb_cache.dat" + def __init__(self, filename, metadata): self.file_depends = metadata.getVar('__depends', False) self.timestamp = bb.parse.cached_mtime(filename) self.variants = self.listvar('__VARIANTS', metadata) + [''] @@ -265,7 +264,7 @@ class Cache(object): return self.has_cache = True - self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") + self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat") logger.debug(1, "Using cache in '%s'", self.cachedir) bb.utils.mkdirhier(self.cachedir) @@ -279,12 +278,21 @@ class Cache(object): old_mtimes.append(newest_mtime) newest_mtime = max(old_mtimes) - if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: + bNeedUpdate = True + if self.caches_array: + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + bNeedUpdate = bNeedUpdate and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime) + cache_class.init_cacheData(self) + if bNeedUpdate: self.load_cachefile() elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") def load_cachefile(self): + # Firstly, using core cache file information for + # valid checking with open(self.cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) try: @@ -301,31 +309,52 @@ class Cache(object): logger.info('Bitbake version mismatch, rebuilding...') return - cachesize = os.fstat(cachefile.fileno()).st_size - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) - previous_percent = 0 - while cachefile: - try: - key = pickled.load() - value = pickled.load() - except Exception: - break + cachesize = 0 + previous_progress = 0 + previous_percent = 0 - self.depends_cache[key] = value - - # only fire events on even percentage boundaries - current_progress = cachefile.tell() - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress), - self.data) - - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + # Calculate the correct cachesize of all those cache files + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + with open(cachefile, "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + while cachefile: + try: + key = pickled.load() + value = pickled.load() + except Exception: + break + if self.depends_cache.has_key(key): + self.depends_cache[key].append(value) + else: + self.depends_cache[key] = [value] + # only fire events on even percentage boundaries + current_progress = cachefile.tell() + previous_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress), + self.data) + + previous_progress += current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, + len(self.depends_cache)), + self.data) + + @staticmethod def virtualfn2realfn(virtualfn): """ @@ -376,8 +405,14 @@ class Cache(object): depends |= (data.getVar("__depends", False) or set()) if depends and not variant: data.setVar("__depends", depends) - info = CoreRecipeInfo(filename, data) - infos.append((virtualfn, info)) + + info_array = [] + for cache_class in caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + info = cache_class(filename, data) + info_array.append(info) + infos.append((virtualfn, info_array)) + return infos def load(self, filename, appends, configdata): @@ -391,8 +426,9 @@ class Cache(object): cached = self.cacheValid(filename) if cached: infos = [] - info = self.depends_cache[filename] - for variant in info.variants: + # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] + info_array = self.depends_cache[filename] + for variant in info_array[0].variants: virtualfn = self.realfn2virtual(filename, variant) infos.append((virtualfn, self.depends_cache[virtualfn])) else: @@ -408,12 +444,12 @@ class Cache(object): skipped, virtuals = 0, 0 cached, infos = self.load(fn, appends, cfgData) - for virtualfn, info in infos: - if info.skipped: + for virtualfn, info_array in infos: + if info_array[0].skipped: logger.debug(1, "Skipping %s", virtualfn) skipped += 1 else: - self.add_info(virtualfn, info, cacheData, not cached) + self.add_info(virtualfn, info_array, cacheData, not cached) virtuals += 1 return cached, skipped, virtuals @@ -457,15 +493,15 @@ class Cache(object): self.remove(fn) return False - info = self.depends_cache[fn] + info_array = self.depends_cache[fn] # Check the file's timestamp - if mtime != info.timestamp: + if mtime != info_array[0].timestamp: logger.debug(2, "Cache: %s changed", fn) self.remove(fn) return False # Check dependencies are still valid - depends = info.file_depends + depends = info_array[0].file_depends if depends: for f, old_mtime in depends: fmtime = bb.parse.cached_mtime_noerror(f) @@ -483,7 +519,7 @@ class Cache(object): return False invalid = False - for cls in info.variants: + for cls in info_array[0].variants: virtualfn = self.realfn2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: @@ -530,13 +566,30 @@ class Cache(object): logger.debug(2, "Cache is clean, not saving.") return - with open(self.cachefile, "wb") as cachefile: - pickler = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL) - pickler.dump(__cache_version__) - pickler.dump(bb.__version__) - for key, value in self.depends_cache.iteritems(): - pickler.dump(key) - pickler.dump(value) + file_dict = {} + pickler_dict = {} + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class_name = cache_class.__name__ + cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + file_dict[cache_class_name] = open(cachefile, "wb") + pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL) + + pickler_dict['CoreRecipeInfo'].dump(__cache_version__) + pickler_dict['CoreRecipeInfo'].dump(bb.__version__) + + try: + for key, info_array in self.depends_cache.iteritems(): + for info in info_array: + if isinstance(info, RecipeInfoCommon): + cache_class_name = info.__class__.__name__ + pickler_dict[cache_class_name].dump(key) + pickler_dict[cache_class_name].dump(info) + finally: + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class_name = cache_class.__name__ + file_dict[cache_class_name].close() del self.depends_cache @@ -544,17 +597,17 @@ class Cache(object): def mtime(cachefile): return bb.parse.cached_mtime_noerror(cachefile) - def add_info(self, filename, info, cacheData, parsed=None): - if not info.skipped: - cacheData.add_from_recipeinfo(filename, info) + def add_info(self, filename, info_array, cacheData, parsed=None): + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + cacheData.add_from_recipeinfo(filename, info_array) if not self.has_cache: return - if (info.skipped or 'SRCREVINACTION' not in info.pv) and not info.nocache: + if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: if parsed: self.cacheclean = False - self.depends_cache[filename] = info + self.depends_cache[filename] = info_array def add(self, file_name, data, cacheData, parsed=None): """ @@ -562,8 +615,12 @@ class Cache(object): """ realfn = self.virtualfn2realfn(file_name)[0] - info = CoreRecipeInfo(realfn, data) - self.add_info(file_name, info, cacheData, parsed) + + info_array = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + info_array.append(cache_class(realfn, data)) + self.add_info(file_name, info_array, cacheData, parsed) @staticmethod def load_bbfile(bbfile, appends, config): @@ -629,7 +686,10 @@ class CacheData(object): def __init__(self, caches_array): self.caches_array = caches_array - CoreRecipeInfo.init_cacheData(self) + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class.init_cacheData(self) + # Direct cache variables self.task_queues = {} self.preferred = {} @@ -640,7 +700,8 @@ class CacheData(object): self.bbfile_priority = {} self.bbfile_config_priorities = [] - def add_from_recipeinfo(self, fn, info): - info.add_cacheData(self, fn) + def add_from_recipeinfo(self, fn, info_array): + for info in info_array: + info.add_cacheData(self, fn) -- cgit v1.2.3-54-g00ecf