bitbake: cooker: Separate out collections handling code into its own class

The Cooker class is too large and needs to be split up into different
functional units. Splitting out the collections code into its own class
seems like a good place to start to try and disentangle things.

(Bitbake rev: ca1fcbb6e214c155a05328779d3d326e10c5eac0)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 126 insertions, 106 deletions

diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 5c52d85901..da598c9330 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -89,7 +89,6 @@ class BBCooker:
 
     def __init__(self, configuration, server_registration_cb, savedenv={}):
         self.status = None
-        self.appendlist = {}
         self.skiplist = {}
 
         self.server_registration_cb = server_registration_cb
@@ -419,7 +418,7 @@ class BBCooker:
 
         if fn:
             try:
-                envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
+                envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.configuration.data)
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
@@ -698,22 +697,13 @@ class BBCooker:
         print("}", file=tdepends_file)
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
-    def calc_bbfile_priority( self, filename, matched = None ):
-        for _, _, regex, pri in self.status.bbfile_config_priorities:
-            if regex.match(filename):
-                if matched != None:
-                    if not regex in matched:
-                        matched.add(regex)
-                return pri
-        return 0
-
     def show_appends_with_no_recipes( self ):
         recipes = set(os.path.basename(f)
                       for f in self.status.pkg_fn.iterkeys())
         recipes |= set(os.path.basename(f)
                       for f in self.skiplist.iterkeys())
-        appended_recipes = self.appendlist.iterkeys()
-        appends_without_recipes = [self.appendlist[recipe]
+        appended_recipes = self.collection.appendlist.iterkeys()
+        appends_without_recipes = [self.collection.appendlist[recipe]
                                    for recipe in appended_recipes
                                    if recipe not in recipes]
         if appends_without_recipes:
@@ -747,32 +737,8 @@ class BBCooker:
                 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee])
             self.status.preferred[providee] = provider
 
-        # Calculate priorities for each file
-        matched = set()
-        for p in self.status.pkg_fn:
-            realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
-            self.status.bbfile_priority[p] = self.calc_bbfile_priority(realfn, matched)
  
-        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
-        unmatched = set()
-        for _, _, regex, pri in self.status.bbfile_config_priorities:        
-            if not regex in matched:
-                unmatched.add(regex)
-
-        def findmatch(regex):
-            for bbfile in self.appendlist:
-                for append in self.appendlist[bbfile]:
-                    if regex.match(append):
-                        return True
-            return False
-
-        for unmatch in unmatched.copy():
-            if findmatch(unmatch):
-                unmatched.remove(unmatch)
-
-        for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
-            if regex in unmatched:
-                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+        self.status.bbfile_priority = self.collection.collection_priorities(self.status.pkg_fn)
 
     def findCoreBaseFiles(self, subdir, configfile):
         corebase = self.configuration.data.getVar('COREBASE', True) or ""
@@ -1111,7 +1077,9 @@ class BBCooker:
         """
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
-        filelist, masked = self.collect_bbfiles()
+
+        self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+        filelist, masked = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1165,7 +1133,7 @@ class BBCooker:
         self.buildSetVars()
 
         self.status = bb.cache.CacheData(self.caches_array)
-        infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \
+        infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
                                      self.configuration.data,
                                      self.caches_array)
         infos = dict(infos)
@@ -1332,7 +1300,9 @@ class BBCooker:
             for dep in self.configuration.extra_assume_provided:
                 self.status.ignored_dependencies.add(dep)
 
-            (filelist, masked) = self.collect_bbfiles()
+            self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+            (filelist, masked) = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
+
             self.configuration.data.renameVar("__depends", "__base_depends")
 
             self.parser = CookerParser(self, filelist, masked)
@@ -1369,7 +1339,87 @@ class BBCooker:
 
         return pkgs_to_build
 
-    def get_bbfiles( self, path = os.getcwd() ):
+
+
+
+    def pre_serve(self):
+        # Empty the environment. The environment will be populated as
+        # necessary from the data store.
+        #bb.utils.empty_environment()
+        try:
+            prserv.serv.auto_start(self.configuration.data)
+        except prserv.serv.PRServiceConfigError:
+            bb.event.fire(CookerExit(), self.configuration.event_data)
+        return
+
+    def post_serve(self):
+        prserv.serv.auto_shutdown(self.configuration.data)
+        bb.event.fire(CookerExit(), self.configuration.event_data)
+
+    def shutdown(self):
+        self.state = state.shutdown
+
+    def stop(self):
+        self.state = state.stop
+
+    def reparseFiles(self):
+        return
+
+    def initialize(self):
+        self.state = state.initial
+        self.initConfigurationData()
+
+    def reset(self):
+        self.state = state.initial
+        self.loadConfigurationData()
+
+def server_main(cooker, func, *args):
+    cooker.pre_serve()
+
+    if cooker.configuration.profile:
+        try:
+            import cProfile as profile
+        except:
+            import profile
+        prof = profile.Profile()
+
+        ret = profile.Profile.runcall(prof, func, *args)
+
+        prof.dump_stats("profile.log")
+        bb.utils.process_profilelog("profile.log")
+        print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
+
+    else:
+        ret = func(*args)
+
+    cooker.post_serve()
+
+    return ret
+
+class CookerExit(bb.event.Event):
+    """
+    Notify clients of the Cooker shutdown
+    """
+
+    def __init__(self):
+        bb.event.Event.__init__(self)
+
+
+class CookerCollectFiles(object):
+    def __init__(self, priorities):
+        self.appendlist = {}
+        self.bbfile_config_priorities = priorities
+
+    def calc_bbfile_priority( self, filename, matched = None ):
+        for _, _, regex, pri in self.bbfile_config_priorities:
+            if regex.match(filename):
+                if matched != None:
+                    if not regex in matched:
+                        matched.add(regex)
+                return pri
+        return 0
+
+    def get_bbfiles(self, path = os.getcwd()):
         """Get list of default .bb files by reading out the current directory"""
         contents = os.listdir(path)
         bbfiles = []
@@ -1379,7 +1429,7 @@ class BBCooker:
                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
         return bbfiles
 
-    def find_bbfiles( self, path ):
+    def find_bbfiles(self, path):
         """Find all the .bb and .bbappend files in a directory"""
         from os.path import join
 
@@ -1392,14 +1442,14 @@ class BBCooker:
 
         return found
 
-    def collect_bbfiles( self ):
+    def collect_bbfiles(self, config, eventdata):
         """Collect all available .bb build files"""
         masked = 0
 
         collectlog.debug(1, "collecting .bb files")
 
-        files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
-        data.setVar("BBFILES", " ".join(files), self.configuration.data)
+        files = (config.getVar( "BBFILES", True) or "").split()
+        config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
         files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
@@ -1409,7 +1459,7 @@ class BBCooker:
 
         if not len(files):
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
-            bb.event.fire(CookerExit(), self.configuration.event_data)
+            bb.event.fire(CookerExit(), eventdata)
 
         # Can't use set here as order is important
         newfiles = []
@@ -1427,7 +1477,7 @@ class BBCooker:
                     if g not in newfiles:
                         newfiles.append(g)
 
-        bbmask = self.configuration.data.getVar('BBMASK', True)
+        bbmask = config.getVar('BBMASK', True)
 
         if bbmask:
             try:
@@ -1475,74 +1525,44 @@ class BBCooker:
     def get_file_appends(self, fn):
         """
         Returns a list of .bbappend files to apply to fn
-        NB: collect_bbfiles() must have been called prior to this
         """
         f = os.path.basename(fn)
         if f in self.appendlist:
             return self.appendlist[f]
         return []
 
-    def pre_serve(self):
-        # Empty the environment. The environment will be populated as
-        # necessary from the data store.
-        #bb.utils.empty_environment()
-        try:
-            prserv.serv.auto_start(self.configuration.data)
-        except prserv.serv.PRServiceConfigError:
-            bb.event.fire(CookerExit(), self.configuration.event_data)
-        return
-
-    def post_serve(self):
-        prserv.serv.auto_shutdown(self.configuration.data)
-        bb.event.fire(CookerExit(), self.configuration.event_data)
-
-    def shutdown(self):
-        self.state = state.shutdown
+    def collection_priorities(self, pkgfns):
 
-    def stop(self):
-        self.state = state.stop
-
-    def reparseFiles(self):
-        return
-
-    def initialize(self):
-        self.state = state.initial
-        self.initConfigurationData()
-
-    def reset(self):
-        self.state = state.initial
-        self.loadConfigurationData()
+        priorities = {}
 
-def server_main(cooker, func, *args):
-    cooker.pre_serve()
-
-    if cooker.configuration.profile:
-        try:
-            import cProfile as profile
-        except:
-            import profile
-        prof = profile.Profile()
-
-        ret = profile.Profile.runcall(prof, func, *args)
-
-        prof.dump_stats("profile.log")
-        bb.utils.process_profilelog("profile.log")
-        print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
-
-    else:
-        ret = func(*args)
+        # Calculate priorities for each file
+        matched = set()
+        for p in pkgfns:
+            realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
+            priorities[p] = self.calc_bbfile_priority(realfn, matched)
+ 
+        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
+        unmatched = set()
+        for _, _, regex, pri in self.bbfile_config_priorities:        
+            if not regex in matched:
+                unmatched.add(regex)
 
-    cooker.post_serve()
+        def findmatch(regex):
+            for bbfile in self.appendlist:
+                for append in self.appendlist[bbfile]:
+                    if regex.match(append):
+                        return True
+            return False
 
-    return ret
+        for unmatch in unmatched.copy():
+            if findmatch(unmatch):
+                unmatched.remove(unmatch)
 
-class CookerExit(bb.event.Event):
-    """
-    Notify clients of the Cooker shutdown
-    """
+        for collection, pattern, regex, _ in self.bbfile_config_priorities:
+            if regex in unmatched:
+                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
-    def __init__(self):
-        bb.event.Event.__init__(self)
+        return priorities
 
 def catch_parse_error(func):
     """Exception handling bits for our parsing"""
@@ -1677,7 +1697,7 @@ class CookerParser(object):
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
-            appends = self.cooker.get_file_appends(filename)
+            appends = self.cooker.collection.get_file_appends(filename)
             if not self.bb_cache.cacheValid(filename, appends):
                 self.willparse.append((filename, appends, cooker.caches_array))
             else:
@@ -1840,7 +1860,7 @@ class CookerParser(object):
 
     def reparse(self, filename):
         infos = self.bb_cache.parse(filename,
-                                    self.cooker.get_file_appends(filename),
+                                    self.cooker.collection.get_file_appends(filename),
                                     self.cfgdata, self.cooker.caches_array)
         for vfn, info_array in infos:
             self.cooker.status.add_from_recipeinfo(vfn, info_array)