1 files changed, 196 insertions, 173 deletions

diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 707d81c971..3e2f698cf7 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -30,7 +30,7 @@
 
 import os
 import logging
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 import bb.data
 import bb.utils
 
@@ -43,7 +43,104 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "132"
+__cache_version__ = "133"
+
+recipe_fields = (
+    'pn',
+    'pv',
+    'pr',
+    'pe',
+    'defaultpref',
+    'depends',
+    'provides',
+    'task_deps',
+    'stamp',
+    'broken',
+    'not_world',
+    'skipped',
+    'timestamp',
+    'packages',
+    'packages_dynamic',
+    'rdepends',
+    'rdepends_pkg',
+    'rprovides',
+    'rprovides_pkg',
+    'rrecommends',
+    'rrecommends_pkg',
+    'nocache',
+    'variants',
+    'file_depends',
+    'tasks',
+    'basetaskhashes',
+    'hashfilename',
+)
+
+
+class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
+    __slots__ = ()
+
+    @classmethod
+    def listvar(cls, var, metadata):
+        return cls.getvar(var, metadata).split()
+
+    @classmethod
+    def intvar(cls, var, metadata):
+        return int(cls.getvar(var, metadata) or 0)
+
+    @classmethod
+    def depvar(cls, var, metadata):
+        return bb.utils.explode_deps(cls.getvar(var, metadata))
+
+    @classmethod
+    def pkgvar(cls, var, packages, metadata):
+        return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata))
+                    for pkg in packages)
+
+    @classmethod
+    def taskvar(cls, var, tasks, metadata):
+        return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata))
+                    for task in tasks)
+    @classmethod
+    def getvar(cls, var, metadata):
+        return metadata.getVar(var, True) or ''
+
+    @classmethod
+    def from_metadata(cls, filename, metadata):
+        tasks = metadata.getVar('__BBTASKS', False)
+
+        packages = cls.listvar('PACKAGES', metadata)
+        return RecipeInfo(
+            tasks            = tasks,
+            basetaskhashes   = cls.taskvar('BB_BASEHASH', tasks, metadata),
+            hashfilename     = cls.getvar('BB_HASHFILENAME', metadata),
+
+            file_depends     = metadata.getVar('__depends', False),
+            task_deps        = metadata.getVar('_task_deps', False) or
+                               {'tasks': [], 'parents': {}},
+            variants         = cls.listvar('__VARIANTS', metadata) + [''],
+
+            skipped          = cls.getvar('__SKIPPED', metadata),
+            timestamp        = bb.parse.cached_mtime(filename),
+            packages         = packages,
+            pn               = cls.getvar('PN', metadata),
+            pe               = cls.getvar('PE', metadata),
+            pv               = cls.getvar('PV', metadata),
+            pr               = cls.getvar('PR', metadata),
+            nocache          = cls.getvar('__BB_DONT_CACHE', metadata),
+            defaultpref      = cls.intvar('DEFAULT_PREFERENCE', metadata),
+            broken           = cls.getvar('BROKEN', metadata),
+            not_world        = cls.getvar('EXCLUDE_FROM_WORLD', metadata),
+            stamp            = cls.getvar('STAMP', metadata),
+            packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata),
+            depends          = cls.depvar('DEPENDS', metadata),
+            provides         = cls.depvar('PROVIDES', metadata),
+            rdepends         = cls.depvar('RDEPENDS', metadata),
+            rprovides        = cls.depvar('RPROVIDES', metadata),
+            rrecommends      = cls.depvar('RRECOMMENDS', metadata),
+            rprovides_pkg    = cls.pkgvar('RPROVIDES', packages, metadata),
+            rdepends_pkg     = cls.pkgvar('RDEPENDS', packages, metadata),
+            rrecommends_pkg  = cls.pkgvar('RRECOMMENDS', packages, metadata),
+        )
 
 
 class Cache(object):
@@ -99,56 +196,6 @@ class Cache(object):
             if os.path.isfile(self.cachefile):
                 logger.info("Out of date cache found, rebuilding...")
 
-    def getVar(self, var, fn, exp=0):
-        """
-        Gets the value of a variable
-        (similar to getVar in the data class)
-
-        There are two scenarios:
-          1. We have cached data - serve from depends_cache[fn]
-          2. We're learning what data to cache - serve from data
-             backend but add a copy of the data to the cache.
-        """
-        if fn in self.clean:
-            return self.depends_cache[fn].get(var)
-
-        self.depends_cache.setdefault(fn, {})
-
-        if fn != self.data_fn:
-            # We're trying to access data in the cache which doesn't exist
-            # yet setData hasn't been called to setup the right access
-            logger.error("data_fn %s and fn %s don't match", self.data_fn, fn)
-
-        self.cacheclean = False
-        result = bb.data.getVar(var, self.data, exp)
-        if result is not None:
-            self.depends_cache[fn][var] = result
-        return result
-
-    def setData(self, virtualfn, fn, data):
-        """
-        Called to prime bb_cache ready to learn which variables to cache.
-        Will be followed by calls to self.getVar which aren't cached
-        but can be fulfilled from self.data.
-        """
-        self.data_fn = virtualfn
-        self.data = data
-
-        # Make sure __depends makes the depends_cache
-        # If we're a virtual class we need to make sure all our depends are
-        # appended to the depends of fn.
-        depends = self.getVar("__depends", virtualfn) or set()
-        self.depends_cache.setdefault(fn, {})
-        if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]:
-            self.depends_cache[fn]["__depends"] = depends
-        else:
-            self.depends_cache[fn]["__depends"].update(depends)
-
-        # Make sure the variants always make it into the cache too
-        self.getVar('__VARIANTS', virtualfn, True)
-
-        self.depends_cache[virtualfn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
-
     @staticmethod
     def virtualfn2realfn(virtualfn):
         """
@@ -193,38 +240,39 @@ class Cache(object):
         to record the variables accessed.
         Return the cache status and whether the file was skipped when parsed
         """
-        skipped = 0
-        virtuals = 0
+        skipped, virtuals = 0, 0
 
         if fn not in self.checked:
             self.cacheValidUpdate(fn)
 
-        if self.cacheValid(fn):
-            multi = self.getVar('__VARIANTS', fn, True)
-            for cls in (multi or "").split() + [""]:
-                virtualfn = self.realfn2virtual(fn, cls)
-                if self.depends_cache[virtualfn].get("__SKIPPED"):
-                    skipped += 1
-                    logger.debug(1, "Skipping %s", virtualfn)
-                    continue
-                self.handle_data(virtualfn, cacheData)
-                virtuals += 1
-            return True, skipped, virtuals
-
-        logger.debug(1, "Parsing %s", fn)
-
-        bb_data = self.load_bbfile(fn, appends, cfgData)
-
-        for data in bb_data:
-            virtualfn = self.realfn2virtual(fn, data)
-            self.setData(virtualfn, fn, bb_data[data])
-            if self.getVar("__SKIPPED", virtualfn):
-                skipped += 1
+        cached = self.cacheValid(fn)
+        if not cached:
+            self.cacheclean = False
+            logger.debug(1, "Parsing %s", fn)
+            datastores = self.load_bbfile(fn, appends, cfgData)
+            depends = set()
+            for variant, data in sorted(datastores.iteritems(),
+                                        key=lambda i: i[0],
+                                        reverse=True):
+                virtualfn = self.realfn2virtual(fn, variant)
+                depends |= (data.getVar("__depends", False) or set())
+                if depends and not variant:
+                    data.setVar("__depends", depends)
+                info = RecipeInfo.from_metadata(fn, data)
+                self.depends_cache[virtualfn] = info
+
+        info = self.depends_cache[fn]
+        for variant in info.variants:
+            virtualfn = self.realfn2virtual(fn, variant)
+            vinfo = self.depends_cache[virtualfn]
+            if vinfo.skipped:
                 logger.debug(1, "Skipping %s", virtualfn)
+                skipped += 1
             else:
-                self.handle_data(virtualfn, cacheData)
+                cacheData.add_from_recipeinfo(virtualfn, vinfo)
                 virtuals += 1
-        return False, skipped, virtuals
+
+        return cached, skipped, virtuals
 
     def cacheValid(self, fn):
         """
@@ -266,14 +314,15 @@ class Cache(object):
             self.remove(fn)
             return False
 
+        info = self.depends_cache[fn]
         # Check the file's timestamp
-        if mtime != self.getVar("CACHETIMESTAMP", fn, True):
+        if mtime != info.timestamp:
             logger.debug(2, "Cache: %s changed", fn)
             self.remove(fn)
             return False
 
         # Check dependencies are still valid
-        depends = self.getVar("__depends", fn, True)
+        depends = info.file_depends
         if depends:
             for f, old_mtime in depends:
                 fmtime = bb.parse.cached_mtime_noerror(f)
@@ -290,20 +339,17 @@ class Cache(object):
                     self.remove(fn)
                     return False
 
-        self.clean.add(fn)
         invalid = False
-        # Mark extended class data as clean too
-        multi = self.getVar('__VARIANTS', fn, True)
-        for cls in (multi or "").split():
+        for cls in info.variants:
             virtualfn = self.realfn2virtual(fn, cls)
             self.clean.add(virtualfn)
-            if not virtualfn in self.depends_cache:
+            if virtualfn not in self.depends_cache:
                 logger.debug(2, "Cache: %s is not cached", virtualfn)
                 invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
         if invalid:
-            for cls in (multi or "").split():
+            for cls in info.variants:
                 virtualfn = self.realfn2virtual(fn, cls)
                 if virtualfn in self.clean:
                     logger.debug(2, "Cache: Removing %s from cache", virtualfn)
@@ -332,7 +378,6 @@ class Cache(object):
         Save the cache
         Called from the parser when complete (or exiting)
         """
-        import copy
 
         if not self.has_cache:
             return
@@ -345,13 +390,12 @@ class Cache(object):
         version_data['CACHE_VER'] = __cache_version__
         version_data['BITBAKE_VER'] = bb.__version__
 
-        cache_data = copy.copy(self.depends_cache)
-        for fn in self.depends_cache:
-            if '__BB_DONT_CACHE' in self.depends_cache[fn] and self.depends_cache[fn]['__BB_DONT_CACHE']:
+        cache_data = dict(self.depends_cache)
+        for fn, info in self.depends_cache.iteritems():
+            if info.nocache:
                 logger.debug(2, "Not caching %s, marked as not cacheable", fn)
                 del cache_data[fn]
-            elif ('PV' in self.depends_cache[fn] and
-                  'SRCREVINACTION' in self.depends_cache[fn]['PV']):
+            elif info.pv and 'SRCREVINACTION' in info.pv:
                 logger.error("Not caching %s as it had SRCREVINACTION in PV. "
                              "Please report this bug", fn)
                 del cache_data[fn]
@@ -364,90 +408,14 @@ class Cache(object):
     def mtime(cachefile):
         return bb.parse.cached_mtime_noerror(cachefile)
 
-    def handle_data(self, file_name, cacheData):
+    def add(self, file_name, data, cacheData):
         """
         Save data we need into the cache
         """
-
-        pn = self.getVar('PN', file_name, True)
-        pe = self.getVar('PE', file_name, True) or "0"
-        pv = self.getVar('PV', file_name, True)
-        if 'SRCREVINACTION' in pv:
-            logger.info("Found SRCREVINACTION in PV (%s) or %s. Please report this bug.", pv, file_name)
-        pr = self.getVar('PR', file_name, True)
-        dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
-        depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
-        packages = (self.getVar('PACKAGES', file_name, True) or "").split()
-        packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
-        rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
-
-        cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name)
-
-        # build PackageName to FileName lookup table
-        cacheData.pkg_pn[pn].append(file_name)
-
-        cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
-
-        cacheData.tasks[file_name] = self.getVar('__BBTASKS', file_name, True)
-        for t in cacheData.tasks[file_name]:
-            cacheData.basetaskhash[file_name + "." + t] = self.getVar("BB_BASEHASH_task-%s" % t, file_name, True)
-
-        # build FileName to PackageName lookup table
-        cacheData.pkg_fn[file_name] = pn
-        cacheData.pkg_pepvpr[file_name] = (pe, pv, pr)
-        cacheData.pkg_dp[file_name] = dp
-
-        provides = [pn]
-        for provide in (self.getVar("PROVIDES", file_name, True) or "").split():
-            if provide not in provides:
-                provides.append(provide)
-
-        # Build forward and reverse provider hashes
-        # Forward: virtual -> [filenames]
-        # Reverse: PN -> [virtuals]
-        cacheData.fn_provides[file_name] = provides
-        for provide in provides:
-            cacheData.providers[provide].append(file_name)
-            if not provide in cacheData.pn_provides[pn]:
-                cacheData.pn_provides[pn].append(provide)
-
-        for dep in depends:
-            if not dep in cacheData.deps[file_name]:
-                cacheData.deps[file_name].append(dep)
-            if not dep in cacheData.all_depends:
-                cacheData.all_depends.append(dep)
-
-        # Build reverse hash for PACKAGES, so runtime dependencies
-        # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
-        for package in packages:
-            cacheData.packages[package].append(file_name)
-            rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
-
-        for package in packages_dynamic:
-            cacheData.packages_dynamic[package].append(file_name)
-
-        for rprovide in rprovides:
-            cacheData.rproviders[rprovide].append(file_name)
-
-        # Build hash of runtime depends and rececommends
-        rdepends = bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or "")
-        rrecommends = bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or "")
-        for package in packages + [pn]:
-            rdeps_pkg = bb.utils.explode_deps(self.getVar('RDEPENDS_%s' % package, file_name, True) or "")
-            cacheData.rundeps[file_name][package] = rdepends + rdeps_pkg
-            rrecs_pkg = bb.utils.explode_deps(self.getVar('RDEPENDS_%s' % package, file_name, True) or "")
-            cacheData.runrecs[file_name][package] = rrecommends + rrecs_pkg
-
-        # Collect files we may need for possible world-dep
-        # calculations
-        if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
-            cacheData.possible_world.append(file_name)
-
-        cacheData.hashfn[file_name] = self.getVar('BB_HASHFILENAME', file_name, True)
-
-        # Touch this to make sure its in the cache
-        self.getVar('__BB_DONT_CACHE', file_name, True)
-        self.getVar('__VARIANTS', file_name, True)
+        realfn = self.virtualfn2realfn(file_name)[0]
+        info = RecipeInfo.from_metadata(realfn, data)
+        self.depends_cache[file_name] = info
+        cacheData.add_from_recipeinfo(file_name, info)
 
     @staticmethod
     def load_bbfile(bbfile, appends, config):
@@ -514,7 +482,6 @@ class CacheData(object):
     def __init__(self):
         """
         Direct cache variables
-        (from Cache.handle_data)
         """
         self.providers = defaultdict(list)
         self.rproviders = defaultdict(list)
@@ -547,3 +514,59 @@ class CacheData(object):
         self.world_target = set()
         self.bbfile_priority = {}
         self.bbfile_config_priorities = []
+
+    def add_from_recipeinfo(self, fn, info):
+        self.task_deps[fn] = info.task_deps
+        self.pkg_fn[fn] = info.pn
+        self.pkg_pn[info.pn].append(fn)
+        self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr)
+        self.pkg_dp[fn] = info.defaultpref
+        self.stamp[fn] = info.stamp
+
+        provides = [info.pn]
+        for provide in info.provides:
+            if provide not in provides:
+                provides.append(provide)
+        self.fn_provides[fn] = provides
+
+        for provide in provides:
+            self.providers[provide].append(fn)
+            if provide not in self.pn_provides[info.pn]:
+                self.pn_provides[info.pn].append(provide)
+
+        for dep in info.depends:
+            if dep not in self.deps[fn]:
+                self.deps[fn].append(dep)
+            if dep not in self.all_depends:
+                self.all_depends.append(dep)
+
+        rprovides = info.rprovides
+        for package in info.packages:
+            self.packages[package].append(fn)
+            rprovides += info.rprovides_pkg[package]
+
+        for package in info.packages_dynamic:
+            self.packages_dynamic[package].append(fn)
+
+        for rprovide in rprovides:
+            self.rproviders[rprovide].append(fn)
+
+        # Build hash of runtime depends and rececommends
+        for package in info.packages + [info.pn]:
+            rundeps, runrecs = list(info.rdepends), list(info.rrecommends)
+            if package in info.packages:
+                rundeps += info.rdepends_pkg[package]
+                runrecs += info.rrecommends_pkg[package]
+            self.rundeps[fn][package] = rundeps
+            self.runrecs[fn][package] = runrecs
+
+        # Collect files we may need for possible world-dep
+        # calculations
+        if not info.broken and not info.not_world:
+            self.possible_world.append(fn)
+
+        self.hashfn[fn] = info.hashfilename
+
+        for task, taskhash in info.basetaskhashes.iteritems():
+            identifier = '%s.%s' % (fn, task)
+            self.basetaskhash[identifier] = taskhash