bitbake: Upgrade from 1.4 -> 1.7.4ish

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@863 311d38ba-8fff-0310-9ca6-ca027cbcb966

1 files changed, 166 insertions, 41 deletions

diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 921a9f7589..05c42518a7 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -33,15 +33,15 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
 import os, re
 import bb.data
 import bb.utils
+from sets import Set
 
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
-    print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
+    bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
 
-# __cache_version__ = "123"
-__cache_version__ = "124" # changes the __depends structure
+__cache_version__ = "125"
 
 class Cache:
     """
@@ -58,14 +58,12 @@ class Cache:
 
         if self.cachedir in [None, '']:
             self.has_cache = False
-            if cooker.cb is not None:
-                print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
+            bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.")
         else:
             self.has_cache = True
             self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
             
-            if cooker.cb is not None:
-                print "NOTE: Using cache in '%s'" % self.cachedir
+            bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
             try:
                 os.stat( self.cachedir )
             except OSError:
@@ -80,7 +78,7 @@ class Cache:
                 if version_data['BITBAKE_VER'] != bb.__version__:
                     raise ValueError, 'Bitbake Version Mismatch'
             except (ValueError, KeyError):
-                bb.note("Invalid cache found, rebuilding...")
+                bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
                 self.depends_cache = {}
 
         if self.depends_cache:
@@ -108,7 +106,7 @@ class Cache:
         if fn != self.data_fn:
             # We're trying to access data in the cache which doesn't exist
             # yet setData hasn't been called to setup the right access. Very bad.
-            bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
+            bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
 
         result = bb.data.getVar(var, self.data, exp)
         self.depends_cache[fn][var] = result
@@ -127,15 +125,15 @@ class Cache:
         self.getVar("__depends", fn, True)
         self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
 
-    def loadDataFull(self, fn, cooker):
+    def loadDataFull(self, fn, cfgData):
         """
         Return a complete set of data for fn.
         To do this, we need to parse the file.
         """
-        bb_data, skipped = self.load_bbfile(fn, cooker)
+        bb_data, skipped = self.load_bbfile(fn, cfgData)
         return bb_data
 
-    def loadData(self, fn, cooker):
+    def loadData(self, fn, cfgData):
         """
         Load a subset of data for fn.
         If the cached data is valid we do nothing,
@@ -148,7 +146,7 @@ class Cache:
                 return True, True
             return True, False
 
-        bb_data, skipped = self.load_bbfile(fn, cooker)
+        bb_data, skipped = self.load_bbfile(fn, cfgData)
         self.setData(fn, bb_data)
         return False, skipped
 
@@ -175,32 +173,36 @@ class Cache:
 
         # Check file still exists
         if self.mtime(fn) == 0:
-            bb.debug(2, "Cache: %s not longer exists" % fn)
+            bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
             self.remove(fn)
             return False
 
         # File isn't in depends_cache
         if not fn in self.depends_cache:
-            bb.debug(2, "Cache: %s is not cached" % fn)
+            bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn)
             self.remove(fn)
             return False
 
         # Check the file's timestamp
         if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
-            bb.debug(2, "Cache: %s changed" % fn)
+            bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
             self.remove(fn)
             return False
 
         # Check dependencies are still valid
         depends = self.getVar("__depends", fn, True)
         for f,old_mtime in depends:
+            # Check if file still exists
+            if self.mtime(f) == 0:
+                return False
+
             new_mtime = bb.parse.cached_mtime(f)
             if (new_mtime > old_mtime):
-                bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f))
+                bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
                 self.remove(fn)
                 return False
 
-        bb.debug(2, "Depends Cache: %s is clean" % fn)
+        bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
         if not fn in self.clean:
             self.clean[fn] = ""
 
@@ -220,7 +222,7 @@ class Cache:
         Remove a fn from the cache
         Called from the parser in error cases
         """
-        bb.debug(1, "Removing %s from cache" % fn)
+        bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn)
         if fn in self.depends_cache:
             del self.depends_cache[fn]
         if fn in self.clean:
@@ -229,7 +231,7 @@ class Cache:
     def sync(self):
         """
         Save the cache
-        Called from the parser when complete (or exitting)
+        Called from the parser when complete (or exiting)
         """
 
         if not self.has_cache:
@@ -243,12 +245,103 @@ class Cache:
         p.dump([self.depends_cache, version_data])
 
     def mtime(self, cachefile):
-        try:
-            return os.stat(cachefile)[8]
-        except OSError:
-            return 0
+        return bb.parse.cached_mtime_noerror(cachefile)
 
-    def load_bbfile( self, bbfile , cooker):
+    def handle_data(self, file_name, cacheData):
+        """
+        Save data we need into the cache 
+        """
+
+        pn       = self.getVar('PN', file_name, True)
+        pv       = self.getVar('PV', file_name, True)
+        pr       = self.getVar('PR', file_name, True)
+        dp       = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
+        provides  = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
+        depends   = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
+        packages  = (self.getVar('PACKAGES', file_name, True) or "").split()
+        packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
+        rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
+
+        cacheData.task_queues[file_name] = self.getVar("_task_graph", file_name, True)
+        cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True)
+
+        # build PackageName to FileName lookup table
+        if pn not in cacheData.pkg_pn:
+            cacheData.pkg_pn[pn] = []
+        cacheData.pkg_pn[pn].append(file_name)
+
+        cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
+
+        # build FileName to PackageName lookup table
+        cacheData.pkg_fn[file_name] = pn
+        cacheData.pkg_pvpr[file_name] = (pv,pr)
+        cacheData.pkg_dp[file_name] = dp
+
+        # Build forward and reverse provider hashes
+        # Forward: virtual -> [filenames]
+        # Reverse: PN -> [virtuals]
+        if pn not in cacheData.pn_provides:
+            cacheData.pn_provides[pn] = Set()
+        cacheData.pn_provides[pn] |= provides
+
+        for provide in provides:
+            if provide not in cacheData.providers:
+                cacheData.providers[provide] = []
+            cacheData.providers[provide].append(file_name)
+
+        cacheData.deps[file_name] = Set()
+        for dep in depends:
+            cacheData.all_depends.add(dep)
+            cacheData.deps[file_name].add(dep)
+
+        # Build reverse hash for PACKAGES, so runtime dependencies 
+        # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
+        for package in packages:
+            if not package in cacheData.packages:
+                cacheData.packages[package] = []
+            cacheData.packages[package].append(file_name)
+            rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() 
+
+        for package in packages_dynamic:
+            if not package in cacheData.packages_dynamic:
+                cacheData.packages_dynamic[package] = []
+            cacheData.packages_dynamic[package].append(file_name)
+
+        for rprovide in rprovides:
+            if not rprovide in cacheData.rproviders:
+                cacheData.rproviders[rprovide] = []
+            cacheData.rproviders[rprovide].append(file_name)
+
+        # Build hash of runtime depends and rececommends
+
+        def add_dep(deplist, deps):
+            for dep in deps:
+                if not dep in deplist:
+                    deplist[dep] = ""
+
+        if not file_name in cacheData.rundeps:
+            cacheData.rundeps[file_name] = {}
+        if not file_name in cacheData.runrecs:
+            cacheData.runrecs[file_name] = {}
+
+        for package in packages + [pn]:
+            if not package in cacheData.rundeps[file_name]:
+                cacheData.rundeps[file_name][package] = {}
+            if not package in cacheData.runrecs[file_name]:
+                cacheData.runrecs[file_name][package] = {}
+
+            add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or ""))
+            add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or ""))
+            add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
+            add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
+
+        # Collect files we may need for possible world-dep
+        # calculations
+        if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
+            cacheData.possible_world.append(file_name)
+
+
+    def load_bbfile( self, bbfile , config):
         """
         Load and parse one .bb build file
         Return the data and whether parsing resulted in the file being skipped
@@ -257,25 +350,15 @@ class Cache:
         import bb
         from bb import utils, data, parse, debug, event, fatal
 
-        topdir = data.getVar('TOPDIR', cooker.configuration.data)
-        if not topdir:
-            topdir = os.path.abspath(os.getcwd())
-            # set topdir to here
-            data.setVar('TOPDIR', topdir, cooker.configuration)
-        bbfile = os.path.abspath(bbfile)
-        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
         # expand tmpdir to include this topdir
-        data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data)
-        # set topdir to location of .bb file
-        topdir = bbfile_loc
-        #data.setVar('TOPDIR', topdir, cfg)
-        # go there
+        data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
+        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
         oldpath = os.path.abspath(os.getcwd())
-        if self.mtime(topdir):
-            os.chdir(topdir)
-        bb_data = data.init_db(cooker.configuration.data)
+        if self.mtime(bbfile_loc):
+            os.chdir(bbfile_loc)
+        bb_data = data.init_db(config)
         try:
-            parse.handle(bbfile, bb_data) # read .bb data
+            bb_data = parse.handle(bbfile, bb_data) # read .bb data
             os.chdir(oldpath)
             return bb_data, False
         except bb.parse.SkipPackage:
@@ -304,3 +387,45 @@ def init(cooker):
     """
     return Cache(cooker)
 
+
+
+#============================================================================#
+# CacheData
+#============================================================================#
+class CacheData:
+    """
+    The data structures we compile from the cached data
+    """
+
+    def __init__(self):
+        """
+        Direct cache variables
+        (from Cache.handle_data)
+        """
+        self.providers   = {}
+        self.rproviders = {}
+        self.packages = {}
+        self.packages_dynamic = {}
+        self.possible_world = []
+        self.pkg_pn = {}
+        self.pkg_fn = {}
+        self.pkg_pvpr = {}
+        self.pkg_dp = {}
+        self.pn_provides = {}
+        self.all_depends = Set()
+        self.deps = {}
+        self.rundeps = {}
+        self.runrecs = {}
+        self.task_queues = {}
+        self.task_deps = {}
+        self.stamp = {}
+        self.preferred = {}
+
+        """
+        Indirect Cache variables
+        (set elsewhere)
+        """
+        self.ignored_dependencies = []
+        self.world_target = Set()
+        self.bbfile_priority = {}
+        self.bbfile_config_priorities = []