1 files changed, 108 insertions, 74 deletions

diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2406dfe95b..488bc610d2 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -22,11 +22,13 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import sys, os, getopt, glob, copy, os.path, re, time
+from __future__ import print_function
+import sys, os, glob, os.path, re, time
+import sre_constants
+from cStringIO import StringIO
+from contextlib import closing
 import bb
-from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
-from bb import command
-import itertools, sre_constants
+from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
 
 class MultipleMatches(Exception):
     """
@@ -121,11 +123,11 @@ class BBCooker:
             self.commandlineAction = None
 
             if 'world' in self.configuration.pkgs_to_build:
-                bb.error("'world' is not a valid target for --environment.")
+                bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.")
             elif len(self.configuration.pkgs_to_build) > 1:
-                bb.error("Only one target can be used with the --environment option.")
+                bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.")
             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
-                bb.error("No target should be used with the --environment and --buildfile options.")
+                bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.")
             elif len(self.configuration.pkgs_to_build) > 0:
                 self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
             else:
@@ -138,21 +140,18 @@ class BBCooker:
             self.commandlineAction = ["showVersions"]
         elif self.configuration.parse_only:
             self.commandlineAction = ["parseFiles"]
-        # FIXME - implement
-        #elif self.configuration.interactive:
-        #    self.interactiveMode()
         elif self.configuration.dot_graph:
             if self.configuration.pkgs_to_build:
                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
             else:
                 self.commandlineAction = None
-                bb.error("Please specify a package name for dependency graph generation.")
+                bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.")
         else:
             if self.configuration.pkgs_to_build:
                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
             else:
                 self.commandlineAction = None
-                bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+                bb.msg.error(bb.msg.domain.Build, "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
 
     def runCommands(self, server, data, abort):
         """
@@ -174,14 +173,14 @@ class BBCooker:
         except bb.build.FuncFailed:
             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
             raise
-        except bb.build.EventException, e:
+        except bb.build.EventException as e:
             event = e.args[1]
             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
             raise
 
     def tryBuild(self, fn, task):
         """
-        Build a provider and its dependencies. 
+        Build a provider and its dependencies.
         build_depends is a list of previous build dependencies (not runtime)
         If build_depends is empty, we're dealing with a runtime depends
         """
@@ -206,7 +205,7 @@ class BBCooker:
 
         # Sort by priority
         for pn in pkg_pn:
-            (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
+            (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
             preferred_versions[pn] = (pref_ver, pref_file)
             latest_versions[pn] = (last_ver, last_file)
 
@@ -260,27 +259,22 @@ class BBCooker:
         if fn:
             try:
                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
-            except IOError, e:
+            except IOError as e:
                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
                 raise
-            except Exception, e:
+            except Exception as e:
                 bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
                 raise
 
-        class dummywrite:
-            def __init__(self):
-                self.writebuf = ""
-            def write(self, output):
-                self.writebuf = self.writebuf + output
-
         # emit variables and shell functions
         try:
             data.update_data(envdata)
-            wb = dummywrite()
-            data.emit_env(wb, envdata, True)
-            bb.msg.plain(wb.writebuf)
-        except Exception, e:
+            with closing(StringIO()) as env:
+                data.emit_env(env, envdata, True)
+                bb.msg.plain(env.getvalue())
+        except Exception as e:
             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
+
         # emit the metadata which isnt valid shell
         data.expandKeys(envdata)
         for e in envdata.keys():
@@ -315,7 +309,7 @@ class BBCooker:
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
         rq.prepare_runqueue()
 
-        seen_fnids = []  
+        seen_fnids = []
         depend_tree = {}
         depend_tree["depends"] = {}
         depend_tree["tdepends"] = {}
@@ -352,7 +346,7 @@ class BBCooker:
 
                 depend_tree["rdepends-pn"][pn] = []
                 for rdep in taskdata.rdepids[fnid]:
-                        depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+                    depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
 
                 rdepends = self.status.rundeps[fn]
                 for package in rdepends:
@@ -397,51 +391,51 @@ class BBCooker:
 
         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
         depends_file = file('pn-depends.dot', 'w' )
-        print >> depends_file, "digraph depends {"
+        print("digraph depends {", file=depends_file)
         for pn in depgraph["pn"]:
             fn = depgraph["pn"][pn]["filename"]
             version = depgraph["pn"][pn]["version"]
-            print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+            print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
         for pn in depgraph["depends"]:
             for depend in depgraph["depends"][pn]:
-                print >> depends_file, '"%s" -> "%s"' % (pn, depend)
+                print('"%s" -> "%s"' % (pn, depend), file=depends_file)
         for pn in depgraph["rdepends-pn"]:
             for rdepend in depgraph["rdepends-pn"][pn]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
-        print >> depends_file,  "}"
+                print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
+        print("}", file=depends_file)
         bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
 
         depends_file = file('package-depends.dot', 'w' )
-        print >> depends_file, "digraph depends {"
+        print("digraph depends {", file=depends_file)
         for package in depgraph["packages"]:
             pn = depgraph["packages"][package]["pn"]
             fn = depgraph["packages"][package]["filename"]
             version = depgraph["packages"][package]["version"]
             if package == pn:
-                print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+                print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
             else:
-                print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
+                print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
             for depend in depgraph["depends"][pn]:
-                print >> depends_file, '"%s" -> "%s"' % (package, depend)
+                print('"%s" -> "%s"' % (package, depend), file=depends_file)
         for package in depgraph["rdepends-pkg"]:
             for rdepend in depgraph["rdepends-pkg"][package]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
+                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
         for package in depgraph["rrecs-pkg"]:
             for rdepend in depgraph["rrecs-pkg"][package]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
-        print >> depends_file,  "}"
+                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
+        print("}", file=depends_file)
         bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
 
         tdepends_file = file('task-depends.dot', 'w' )
-        print >> tdepends_file, "digraph depends {"
+        print("digraph depends {", file=tdepends_file)
         for task in depgraph["tdepends"]:
             (pn, taskname) = task.rsplit(".", 1)
             fn = depgraph["pn"][pn]["filename"]
             version = depgraph["pn"][pn]["version"]
-            print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
+            print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
             for dep in depgraph["tdepends"][task]:
-                print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
-        print >> tdepends_file,  "}"
+                print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
+        print("}", file=tdepends_file)
         bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
 
     def buildDepgraph( self ):
@@ -452,9 +446,12 @@ class BBCooker:
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
 
+        matched = set()
         def calc_bbfile_priority(filename):
-            for (regex, pri) in self.status.bbfile_config_priorities:
+            for _, _, regex, pri in self.status.bbfile_config_priorities:
                 if regex.match(filename):
+                    if not regex in matched:
+                        matched.add(regex)
                     return pri
             return 0
 
@@ -473,6 +470,11 @@ class BBCooker:
         for p in self.status.pkg_fn:
             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
 
+        for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
+            if not regex in matched:
+                bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" %
+                                                    (collection, pattern))
+
     def buildWorldTargetList(self):
         """
          Build package list for "bitbake world"
@@ -505,31 +507,57 @@ class BBCooker:
         """Drop off into a shell"""
         try:
             from bb import shell
-        except ImportError, details:
+        except ImportError as details:
             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
         else:
             shell.start( self )
 
+    def _findLayerConf(self):
+        path = os.getcwd()
+        while path != "/":
+            bblayers = os.path.join(path, "conf", "bblayers.conf")
+            if os.path.exists(bblayers):
+                return bblayers
+
+            path, _ = os.path.split(path)
+
     def parseConfigurationFiles(self, files):
         try:
             data = self.configuration.data
             for f in files:
                 data = bb.parse.handle(f, data)
 
-            layerconf = os.path.join(os.getcwd(), "conf", "bblayers.conf")
-            if os.path.exists(layerconf):
+            layerconf = self._findLayerConf()
+            if layerconf:
                 bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
                 data = bb.parse.handle(layerconf, data)
 
                 layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
 
+                data = bb.data.createCopy(data)
                 for layer in layers:
                     bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
                     bb.data.setVar('LAYERDIR', layer, data)
                     data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
 
+                    # XXX: Hack, relies on the local keys of the datasmart
+                    # instance being stored in the 'dict' attribute and makes
+                    # assumptions about how variable expansion works, but
+                    # there's no better way to force an expansion of a single
+                    # variable across the datastore today, and this at least
+                    # lets us reference LAYERDIR without having to immediately
+                    # eval all our variables that use it.
+                    for key in data.dict:
+                        if key != "_data":
+                            value = data.getVar(key, False)
+                            if value and "${LAYERDIR}" in value:
+                                data.setVar(key, value.replace("${LAYERDIR}", layer))
+
                 bb.data.delVar('LAYERDIR', data)
 
+            if not data.getVar("BBPATH", True):
+                bb.fatal("The BBPATH variable is not set")
+
             data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
 
             self.configuration.data = data
@@ -541,16 +569,17 @@ class BBCooker:
 
             # Nomally we only register event handlers at the end of parsing .bb files
             # We register any handlers we've found so far here...
-            for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
-                bb.event.register(var,bb.data.getVar(var, self.configuration.data))
+            for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []:
+                bb.event.register(var, bb.data.getVar(var, self.configuration.data))
 
             bb.fetch.fetcher_init(self.configuration.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
 
-        except IOError, e:
+
+        except IOError as e:
             bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
-        except bb.parse.ParseError, details:
+        except bb.parse.ParseError as details:
             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
 
     def handleCollections( self, collections ):
@@ -573,7 +602,7 @@ class BBCooker:
                     continue
                 try:
                     pri = int(priority)
-                    self.status.bbfile_config_priorities.append((cre, pri))
+                    self.status.bbfile_config_priorities.append((c, regex, cre, pri))
                 except ValueError:
                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
 
@@ -582,8 +611,8 @@ class BBCooker:
         Setup any variables needed before starting a build
         """
         if not bb.data.getVar("BUILDNAME", self.configuration.data):
-            bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
-        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
+            bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data)
+        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data)
 
     def matchFiles(self, buildfile):
         """
@@ -630,13 +659,19 @@ class BBCooker:
         if (task == None):
             task = self.configuration.cmd
 
-        fn = self.matchFile(buildfile)
+        self.bb_cache = bb.cache.init(self)
+        self.status = bb.cache.CacheData()
+
+        (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
+        buildfile = self.matchFile(fn)
+        fn = self.bb_cache.realfn2virtual(buildfile, cls)
+
         self.buildSetVars()
 
         # Load data into the cache for fn and parse the loaded cache data
-        self.bb_cache = bb.cache.init(self)
-        self.status = bb.cache.CacheData()
-        self.bb_cache.loadData(fn, self.configuration.data, self.status)      
+        the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
+        self.bb_cache.setData(fn, buildfile, the_data)
+        self.bb_cache.handle_data(fn, self.status)
 
         # Tweak some variables
         item = self.bb_cache.getVar('PN', fn, True)
@@ -675,8 +710,8 @@ class BBCooker:
             failures = 0
             try:
                 retval = rq.execute_runqueue()
-            except runqueue.TaskFailure, fnids:
-                for fnid in fnids:
+            except runqueue.TaskFailure as exc:
+                for fnid in exc.args:
                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
                     failures = failures + 1
                 retval = False
@@ -711,8 +746,8 @@ class BBCooker:
             failures = 0
             try:
                 retval = rq.execute_runqueue()
-            except runqueue.TaskFailure, fnids:
-                for fnid in fnids:
+            except runqueue.TaskFailure as exc:
+                for fnid in exc.args:
                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
                     failures = failures + 1
                 retval = False
@@ -769,10 +804,10 @@ class BBCooker:
 
             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
             self.status.ignored_dependencies = set(ignore.split())
-    
+
             for dep in self.configuration.extra_assume_provided:
                 self.status.ignored_dependencies.add(dep)
-    
+
             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 
             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
@@ -810,7 +845,7 @@ class BBCooker:
         for f in contents:
             (root, ext) = os.path.splitext(f)
             if ext == ".bb":
-                bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
+                bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
         return bbfiles
 
     def find_bbfiles( self, path ):
@@ -822,7 +857,7 @@ class BBCooker:
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [join(dir,f) for f in files if f.endswith('.bb')]
+            found += [join(dir, f) for f in files if f.endswith('.bb')]
 
         return found
 
@@ -906,9 +941,9 @@ class BBCooker:
             pout.close()
         else:
             self.server.serve_forever()
-        
+
         bb.event.fire(CookerExit(), self.configuration.event_data)
-        
+
 class CookerExit(bb.event.Event):
     """
     Notify clients of the Cooker shutdown
@@ -937,9 +972,9 @@ class CookerParser:
         self.pointer = 0
 
     def parse_next(self):
+        cooker = self.cooker
         if self.pointer < len(self.filelist):
             f = self.filelist[self.pointer]
-            cooker = self.cooker
 
             try:
                 fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
@@ -951,7 +986,7 @@ class CookerParser:
                 self.skipped += skipped
                 self.virtuals += virtuals
 
-            except IOError, e:
+            except IOError as e:
                 self.error += 1
                 cooker.bb_cache.remove(f)
                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
@@ -960,7 +995,7 @@ class CookerParser:
                 cooker.bb_cache.remove(f)
                 cooker.bb_cache.sync()
                 raise
-            except Exception, e:
+            except Exception as e:
                 self.error += 1
                 cooker.bb_cache.remove(f)
                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
@@ -978,4 +1013,3 @@ class CookerParser:
                 raise ParsingErrorsFound
             return False
         return True
-