bitbake: Switch to bitbake-dev version (bitbake master upstream)

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

1 files changed, 482 insertions, 279 deletions

diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 14ccfb59aa..8036d7e9d5 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -7,7 +7,7 @@
 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
 # Copyright (C) 2005        Holger Hans Peter Freyther
 # Copyright (C) 2005        ROAD GmbH
-# Copyright (C) 2006        Richard Purdie
+# Copyright (C) 2006 - 2007 Richard Purdie
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -25,9 +25,35 @@
 import sys, os, getopt, glob, copy, os.path, re, time
 import bb
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
+from bb import command
+import bb.server.xmlrpc
 import itertools, sre_constants
 
-parsespin = itertools.cycle( r'|/-\\' )
+class MultipleMatches(Exception):
+    """
+    Exception raised when multiple file matches are found
+    """
+
+class ParsingErrorsFound(Exception):
+    """
+    Exception raised when parsing errors are found
+    """
+
+class NothingToBuild(Exception):
+    """
+    Exception raised when there is nothing to build
+    """
+
+
+# Different states cooker can be in
+cookerClean = 1
+cookerParsing = 2
+cookerParsed = 3
+
+# Different action states the cooker can be in
+cookerRun = 1           # Cooker is running normally
+cookerShutdown = 2      # Active tasks should be brought to a controlled stop
+cookerStop = 3          # Stop, now!
 
 #============================================================================#
 # BBCooker
@@ -37,12 +63,14 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration):
+    def __init__(self, configuration, server):
         self.status = None
 
         self.cache = None
         self.bb_cache = None
 
+        self.server = server.BitBakeServer(self)
+
         self.configuration = configuration
 
         if self.configuration.verbose:
@@ -58,17 +86,15 @@ class BBCooker:
 
         self.configuration.data = bb.data.init()
 
-    def parseConfiguration(self):
-
         bb.data.inheritFromOS(self.configuration.data)
 
-        # Add conf/bitbake.conf to the list of configuration files to read
-        self.configuration.file.append( os.path.join( "conf", "bitbake.conf" ) )
+        for f in self.configuration.file:
+            self.parseConfigurationFile( f )
 
-        self.parseConfigurationFile(self.configuration.file)
+        self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
 
         if not self.configuration.cmd:
-            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
+            self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
 
         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
@@ -80,9 +106,7 @@ class BBCooker:
         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
         bb.data.update_data(self.configuration.event_data)
 
-        #
         # TOSTOP must not be set or our children will hang when they output
-        #
         fd = sys.stdout.fileno()
         if os.isatty(fd):
             import termios
@@ -92,40 +116,91 @@ class BBCooker:
                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
 
+        self.command = bb.command.Command(self)
+        self.cookerState = cookerClean
+        self.cookerAction = cookerRun
+
+    def parseConfiguration(self):
+
+
         # Change nice level if we're asked to
         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
         if nice:
             curnice = os.nice(0)
             nice = int(nice) - curnice
             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
- 
+
+    def parseCommandLine(self):
+        # Parse any commandline into actions
+        if self.configuration.show_environment:
+            self.commandlineAction = None
+
+            if 'world' in self.configuration.pkgs_to_build:
+                bb.error("'world' is not a valid target for --environment.")
+            elif len(self.configuration.pkgs_to_build) > 1:
+                bb.error("Only one target can be used with the --environment option.")
+            elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
+                bb.error("No target should be used with the --environment and --buildfile options.")
+            elif len(self.configuration.pkgs_to_build) > 0:
+                self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
+            else:
+                self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
+        elif self.configuration.buildfile is not None:
+            self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
+        elif self.configuration.revisions_changed:
+            self.commandlineAction = ["compareRevisions"]
+        elif self.configuration.show_versions:
+            self.commandlineAction = ["showVersions"]
+        elif self.configuration.parse_only:
+            self.commandlineAction = ["parseFiles"]
+        # FIXME - implement
+        #elif self.configuration.interactive:
+        #    self.interactiveMode()
+        elif self.configuration.dot_graph:
+            if self.configuration.pkgs_to_build:
+                self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
+            else:
+                self.commandlineAction = None
+                bb.error("Please specify a package name for dependency graph generation.")
+        else:
+            if self.configuration.pkgs_to_build:
+                self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
+            else:
+                self.commandlineAction = None
+                bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+
+    def runCommands(self, server, data, abort):
+        """
+        Run any queued asynchronous command
+        This is done by the idle handler so it runs in true context rather than
+        tied to any UI.
+        """
+
+        return self.command.runAsyncCommand()
 
     def tryBuildPackage(self, fn, item, task, the_data):
         """
         Build one task of a package, optionally build following task depends
         """
-        bb.event.fire(bb.event.PkgStarted(item, the_data))
         try:
             if not self.configuration.dry_run:
                 bb.build.exec_task('do_%s' % task, the_data)
-            bb.event.fire(bb.event.PkgSucceeded(item, the_data))
             return True
         except bb.build.FuncFailed:
             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
-            bb.event.fire(bb.event.PkgFailed(item, the_data))
             raise
         except bb.build.EventException, e:
             event = e.args[1]
             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
-            bb.event.fire(bb.event.PkgFailed(item, the_data))
             raise
 
-    def tryBuild(self, fn):
+    def tryBuild(self, fn, task):
         """
         Build a provider and its dependencies. 
         build_depends is a list of previous build dependencies (not runtime)
         If build_depends is empty, we're dealing with a runtime depends
         """
+
         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 
         item = self.status.pkg_fn[fn]
@@ -133,9 +208,13 @@ class BBCooker:
         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
         #    return True
 
-        return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data)
+        return self.tryBuildPackage(fn, item, task, the_data)
 
     def showVersions(self):
+
+        # Need files parsed
+        self.updateCache()
+
         pkg_pn = self.status.pkg_pn
         preferred_versions = {}
         latest_versions = {}
@@ -149,43 +228,36 @@ class BBCooker:
         pkg_list = pkg_pn.keys()
         pkg_list.sort()
 
+        bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
+        bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
+
         for p in pkg_list:
             pref = preferred_versions[p]
             latest = latest_versions[p]
 
-            if pref != latest:
-                prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
-            else:
+            prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
+            lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
+
+            if pref == latest:
                 prefstr = ""
 
-            print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
-                                        prefstr)
+            bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
 
+    def compareRevisions(self):
+        ret = bb.fetch.fetcher_compare_revisons(self.configuration.data)
+        bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data)
 
-    def showEnvironment(self , buildfile = None, pkgs_to_build = []):
+    def showEnvironment(self, buildfile = None, pkgs_to_build = []):
         """
         Show the outer or per-package environment
         """
         fn = None
         envdata = None
 
-        if 'world' in pkgs_to_build:
-            print "'world' is not a valid target for --environment."
-            sys.exit(1)
-
-        if len(pkgs_to_build) > 1:
-            print "Only one target can be used with the --environment option."
-            sys.exit(1)
-
         if buildfile:
-            if len(pkgs_to_build) > 0:
-                print "No target should be used with the --environment and --buildfile options."
-                sys.exit(1)
             self.cb = None
             self.bb_cache = bb.cache.init(self)
             fn = self.matchFile(buildfile)
-            if not fn:
-                sys.exit(1)
         elif len(pkgs_to_build) == 1:
             self.updateCache()
 
@@ -193,13 +265,9 @@ class BBCooker:
             bb.data.update_data(localdata)
             bb.data.expandKeys(localdata)
 
-            taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs)
-
-            try:
-                taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
-                taskdata.add_unresolved(localdata, self.status)
-            except bb.providers.NoProvider:
-                sys.exit(1)
+            taskdata = bb.taskdata.TaskData(self.configuration.abort)
+            taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
+            taskdata.add_unresolved(localdata, self.status)
 
             targetid = taskdata.getbuild_id(pkgs_to_build[0])
             fnid = taskdata.build_targets[targetid][0]
@@ -211,55 +279,69 @@ class BBCooker:
             try:
                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
             except IOError, e:
-                bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
+                bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
+                raise
             except Exception, e:
-                bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
+                bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
+                raise
+
+        class dummywrite:
+            def __init__(self):
+                self.writebuf = ""
+            def write(self, output):
+                self.writebuf = self.writebuf + output
 
         # emit variables and shell functions
         try:
-            data.update_data( envdata )
-            data.emit_env(sys.__stdout__, envdata, True)
+            data.update_data(envdata)
+            wb = dummywrite()
+            data.emit_env(wb, envdata, True)
+            bb.msg.plain(wb.writebuf)
         except Exception, e:
             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
         # emit the metadata which isnt valid shell
-        data.expandKeys( envdata )
+        data.expandKeys(envdata)
         for e in envdata.keys():
             if data.getVarFlag( e, 'python', envdata ):
-                sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
+                bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
 
-    def generateDotGraph( self, pkgs_to_build, ignore_deps ):
+    def generateDepTreeData(self, pkgs_to_build, task):
         """
-        Generate a task dependency graph. 
-
-        pkgs_to_build A list of packages that needs to be built
-        ignore_deps   A list of names where processing of dependencies
-                      should be stopped. e.g. dependencies that get
+        Create a dependency tree of pkgs_to_build, returning the data.
         """
 
-        for dep in ignore_deps:
-            self.status.ignored_dependencies.add(dep)
+        # Need files parsed
+        self.updateCache()
+
+        # If we are told to do the None task then query the default task
+        if (task == None):
+            task = self.configuration.cmd
+
+        pkgs_to_build = self.checkPackages(pkgs_to_build)
 
         localdata = data.createCopy(self.configuration.data)
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs)
+        taskdata = bb.taskdata.TaskData(self.configuration.abort)
 
         runlist = []
-        try:
-            for k in pkgs_to_build:
-                taskdata.add_provider(localdata, self.status, k)
-                runlist.append([k, "do_%s" % self.configuration.cmd])
-            taskdata.add_unresolved(localdata, self.status)
-        except bb.providers.NoProvider:
-            sys.exit(1)
+        for k in pkgs_to_build:
+            taskdata.add_provider(localdata, self.status, k)
+            runlist.append([k, "do_%s" % task])
+        taskdata.add_unresolved(localdata, self.status)
+
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
         rq.prepare_runqueue()
 
         seen_fnids = []  
-        depends_file = file('depends.dot', 'w' )
-        tdepends_file = file('task-depends.dot', 'w' )
-        print >> depends_file, "digraph depends {"
-        print >> tdepends_file, "digraph depends {"
+        depend_tree = {}
+        depend_tree["depends"] = {}
+        depend_tree["tdepends"] = {}
+        depend_tree["pn"] = {}
+        depend_tree["rdepends-pn"] = {}
+        depend_tree["packages"] = {}
+        depend_tree["rdepends-pkg"] = {}
+        depend_tree["rrecs-pkg"] = {}
 
         for task in range(len(rq.runq_fnid)):
             taskname = rq.runq_task[task]
@@ -267,43 +349,118 @@ class BBCooker:
             fn = taskdata.fn_index[fnid]
             pn = self.status.pkg_fn[fn]
             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
-            print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
+            if pn not in depend_tree["pn"]:
+                depend_tree["pn"][pn] = {}
+                depend_tree["pn"][pn]["filename"] = fn
+                depend_tree["pn"][pn]["version"] = version
             for dep in rq.runq_depends[task]:
                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
                 deppn = self.status.pkg_fn[depfn]
-                print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
+                dotname = "%s.%s" % (pn, rq.runq_task[task])
+                if not dotname in depend_tree["tdepends"]:
+                    depend_tree["tdepends"][dotname] = []
+                depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
             if fnid not in seen_fnids:
                 seen_fnids.append(fnid)
                 packages = []
-                print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
-                for depend in self.status.deps[fn]:
-                    print >> depends_file, '"%s" -> "%s"' % (pn, depend)
+
+                depend_tree["depends"][pn] = []
+                for dep in taskdata.depids[fnid]:
+                    depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+
+                depend_tree["rdepends-pn"][pn] = []
+                for rdep in taskdata.rdepids[fnid]:
+                        depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+
                 rdepends = self.status.rundeps[fn]
                 for package in rdepends:
-                    for rdepend in re.findall("([\w.-]+)(\ \(.+\))?", rdepends[package]):
-                        print >> depends_file, '"%s" -> "%s%s" [style=dashed]' % (package, rdepend[0], rdepend[1])
+                    depend_tree["rdepends-pkg"][package] = []
+                    for rdepend in rdepends[package]:
+                        depend_tree["rdepends-pkg"][package].append(rdepend)
                     packages.append(package)
+
                 rrecs = self.status.runrecs[fn]
                 for package in rrecs:
-                    for rdepend in re.findall("([\w.-]+)(\ \(.+\))?", rrecs[package]):
-                        print >> depends_file, '"%s" -> "%s%s" [style=dashed]' % (package, rdepend[0], rdepend[1])
+                    depend_tree["rrecs-pkg"][package] = []
+                    for rdepend in rrecs[package]:
+                        depend_tree["rrecs-pkg"][package].append(rdepend)
                     if not package in packages:
                         packages.append(package)
+
                 for package in packages:
-                    if package != pn:
-                        print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
-                        for depend in self.status.deps[fn]:
-                            print >> depends_file, '"%s" -> "%s"' % (package, depend)
-                # Prints a flattened form of the above where subpackages of a package are merged into the main pn
-                #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
-                #for rdep in taskdata.rdepids[fnid]:
-                #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
-                #for dep in taskdata.depids[fnid]:
-                #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
+                    if package not in depend_tree["packages"]:
+                        depend_tree["packages"][package] = {}
+                        depend_tree["packages"][package]["pn"] = pn
+                        depend_tree["packages"][package]["filename"] = fn
+                        depend_tree["packages"][package]["version"] = version
+
+        return depend_tree
+
+
+    def generateDepTreeEvent(self, pkgs_to_build, task):
+        """
+        Create a task dependency graph of pkgs_to_build.
+        Generate an event with the result
+        """
+        depgraph = self.generateDepTreeData(pkgs_to_build, task)
+        bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
+
+    def generateDotGraphFiles(self, pkgs_to_build, task):
+        """
+        Create a task dependency graph of pkgs_to_build.
+        Save the result to a set of .dot files.
+        """
+
+        depgraph = self.generateDepTreeData(pkgs_to_build, task)
+
+        # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
+        depends_file = file('pn-depends.dot', 'w' )
+        print >> depends_file, "digraph depends {"
+        for pn in depgraph["pn"]:
+            fn = depgraph["pn"][pn]["filename"]
+            version = depgraph["pn"][pn]["version"]
+            print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+        for pn in depgraph["depends"]:
+            for depend in depgraph["depends"][pn]:
+                print >> depends_file, '"%s" -> "%s"' % (pn, depend)
+        for pn in depgraph["rdepends-pn"]:
+            for rdepend in depgraph["rdepends-pn"][pn]:
+                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
+        print >> depends_file,  "}"
+        bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
+
+        depends_file = file('package-depends.dot', 'w' )
+        print >> depends_file, "digraph depends {"
+        for package in depgraph["packages"]:
+            pn = depgraph["packages"][package]["pn"]
+            fn = depgraph["packages"][package]["filename"]
+            version = depgraph["packages"][package]["version"]
+            if package == pn:
+                print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+            else:
+                print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
+            for depend in depgraph["depends"][pn]:
+                print >> depends_file, '"%s" -> "%s"' % (package, depend)
+        for package in depgraph["rdepends-pkg"]:
+            for rdepend in depgraph["rdepends-pkg"][package]:
+                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
+        for package in depgraph["rrecs-pkg"]:
+            for rdepend in depgraph["rrecs-pkg"][package]:
+                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
         print >> depends_file,  "}"
+        bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
+
+        tdepends_file = file('task-depends.dot', 'w' )
+        print >> tdepends_file, "digraph depends {"
+        for task in depgraph["tdepends"]:
+            (pn, taskname) = task.rsplit(".", 1)
+            fn = depgraph["pn"][pn]["filename"]
+            version = depgraph["pn"][pn]["version"]
+            print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
+            for dep in depgraph["tdepends"][task]:
+                print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
         print >> tdepends_file,  "}"
-        bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
-        bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
+        bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
 
     def buildDepgraph( self ):
         all_depends = self.status.all_depends
@@ -324,7 +481,7 @@ class BBCooker:
             try:
                 (providee, provider) = p.split(':')
             except:
-                bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
+                bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
                 continue
             if providee in self.status.preferred and self.status.preferred[providee] != provider:
                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
@@ -362,19 +519,6 @@ class BBCooker:
             self.status.possible_world = None
             self.status.all_depends    = None
 
-    def myProgressCallback( self, x, y, f, from_cache ):
-        """Update any tty with the progress change"""
-        if os.isatty(sys.stdout.fileno()):
-            sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
-            sys.stdout.flush()
-        else:
-            if x == 1:
-                sys.stdout.write("Parsing .bb files, please wait...")
-                sys.stdout.flush()
-            if x == y:
-                sys.stdout.write("done.")
-                sys.stdout.flush()
-
     def interactiveMode( self ):
         """Drop off into a shell"""
         try:
@@ -383,12 +527,10 @@ class BBCooker:
             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
         else:
             shell.start( self )
-            sys.exit( 0 )
 
-    def parseConfigurationFile( self, afiles ):
+    def parseConfigurationFile( self, afile ):
         try:
-            for afile in afiles:
-                self.configuration.data = bb.parse.handle( afile, self.configuration.data )
+            self.configuration.data = bb.parse.handle( afile, self.configuration.data )
 
             # Handle any INHERITs and inherit the base class
             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
@@ -402,10 +544,10 @@ class BBCooker:
 
             bb.fetch.fetcher_init(self.configuration.data)
 
-            bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
+            bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
 
         except IOError, e:
-            bb.msg.fatal(bb.msg.domain.Parsing, "IO Error: %s" % str(e) )
+            bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (afile, str(e)))
         except bb.parse.ParseError, details:
             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
 
@@ -439,17 +581,17 @@ class BBCooker:
         """
         if not bb.data.getVar("BUILDNAME", self.configuration.data):
             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
-        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
+        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
 
-    def matchFile(self, buildfile):
+    def matchFiles(self, buildfile):
         """
-        Convert the fragment buildfile into a real file
-        Error if there are too many matches
+        Find the .bb files which match the expression in 'buildfile'.
         """
+
         bf = os.path.abspath(buildfile)
         try:
             os.stat(bf)
-            return bf
+            return [bf]
         except OSError:
             (filelist, masked) = self.collect_bbfiles()
             regexp = re.compile(buildfile)
@@ -458,27 +600,41 @@ class BBCooker:
                 if regexp.search(f) and os.path.isfile(f):
                     bf = f
                     matches.append(f)
-            if len(matches) != 1:
-                bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
-                for f in matches:
-                    bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
-                return False
-            return matches[0]
+            return matches
 
-    def buildFile(self, buildfile):
+    def matchFile(self, buildfile):
+        """
+        Find the .bb file which matches the expression in 'buildfile'.
+        Raise an error if multiple files
+        """
+        matches = self.matchFiles(buildfile)
+        if len(matches) != 1:
+            bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
+            for f in matches:
+                bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
+            raise MultipleMatches
+        return matches[0]
+
+    def buildFile(self, buildfile, task):
         """
         Build the file matching regexp buildfile
         """
 
-        # Make sure our target is a fully qualified filename
+        # Parse the configuration here. We need to do it explicitly here since
+        # buildFile() doesn't use the cache
+        self.parseConfiguration()
+
+        # If we are told to do the None task then query the default task
+        if (task == None):
+            task = self.configuration.cmd
+
         fn = self.matchFile(buildfile)
-        if not fn:
-            return False
+        self.buildSetVars()
 
         # Load data into the cache for fn and parse the loaded cache data
         self.bb_cache = bb.cache.init(self)
         self.status = bb.cache.CacheData()
-        self.bb_cache.loadData(fn, self.configuration.data, self.status)
+        self.bb_cache.loadData(fn, self.configuration.data, self.status)      
 
         # Tweak some variables
         item = self.bb_cache.getVar('PN', fn, True)
@@ -493,159 +649,157 @@ class BBCooker:
 
         # Remove stamp for target if force mode active
         if self.configuration.force:
-            bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn))
-            bb.build.del_stamp('do_%s' % self.configuration.cmd, self.configuration.data)
+            bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
+            bb.build.del_stamp('do_%s' % task, self.status, fn)
 
         # Setup taskdata structure
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs)
+        taskdata = bb.taskdata.TaskData(self.configuration.abort)
         taskdata.add_provider(self.configuration.data, self.status, item)
 
         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
-        bb.event.fire(bb.event.BuildStarted(buildname, [item], self.configuration.event_data))
+        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
 
         # Execute the runqueue
-        runlist = [[item, "do_%s" % self.configuration.cmd]]
+        runlist = [[item, "do_%s" % task]]
+
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
-        rq.prepare_runqueue()
-        try:
-            failures = rq.execute_runqueue()
-        except runqueue.TaskFailure, fnids:
+
+        def buildFileIdle(server, rq, abort):
+
+            if abort or self.cookerAction == cookerStop:
+                rq.finish_runqueue(True)
+            elif self.cookerAction == cookerShutdown:
+                rq.finish_runqueue(False)
             failures = 0
-            for fnid in fnids:
-                bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
-                failures = failures + 1
-            bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures))
-            return False
-        bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures))
-        return True
+            try:
+                retval = rq.execute_runqueue()
+            except runqueue.TaskFailure, fnids:
+                for fnid in fnids:
+                    bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
+                    failures = failures + 1
+                retval = False
+            if not retval:
+                self.command.finishAsyncCommand()
+                bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
+                return False
+            return 0.5
+
+        self.server.register_idle_function(buildFileIdle, rq)
 
-    def buildTargets(self, targets):
+    def buildTargets(self, targets, task):
         """
         Attempt to build the targets specified
         """
 
-        buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
-        bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
+        # Need files parsed
+        self.updateCache()
 
-        localdata = data.createCopy(self.configuration.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
+        # If we are told to do the NULL task then query the default task
+        if (task == None):
+            task = self.configuration.cmd
 
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs)
+        targets = self.checkPackages(targets)
 
-        runlist = []
-        try:
-            for k in targets:
-                taskdata.add_provider(localdata, self.status, k)
-                runlist.append([k, "do_%s" % self.configuration.cmd])
-            taskdata.add_unresolved(localdata, self.status)
-        except bb.providers.NoProvider:
-            sys.exit(1)
+        def buildTargetsIdle(server, rq, abort):
 
-        rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
-        rq.prepare_runqueue()
-        try:
-            failures = rq.execute_runqueue()
-        except runqueue.TaskFailure, fnids:
+            if abort or self.cookerAction == cookerStop:
+                rq.finish_runqueue(True)
+            elif self.cookerAction == cookerShutdown:
+                rq.finish_runqueue(False)
             failures = 0
-            for fnid in fnids:
-                bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
-                failures = failures + 1
-            bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
-            sys.exit(1)
-        bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
+            try:
+                retval = rq.execute_runqueue()
+            except runqueue.TaskFailure, fnids:
+                for fnid in fnids:
+                    bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
+                    failures = failures + 1
+                retval = False
+            if not retval:
+                self.command.finishAsyncCommand()
+                bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
+                return None
+            return 0.5
 
-        sys.exit(0)
+        self.buildSetVars()
 
-    def updateCache(self):
-        # Import Psyco if available and not disabled
-        import platform
-        if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
-            if not self.configuration.disable_psyco:
-                try:
-                    import psyco
-                except ImportError:
-                    bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
-                else:
-                    psyco.bind( self.parse_bbfiles )
-            else:
-                bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
+        buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
+        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
 
-        self.status = bb.cache.CacheData()
+        localdata = data.createCopy(self.configuration.data)
+        bb.data.update_data(localdata)
+        bb.data.expandKeys(localdata)
 
-        ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
-        self.status.ignored_dependencies = set( ignore.split() )
+        taskdata = bb.taskdata.TaskData(self.configuration.abort)
 
-        self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
+        runlist = []
+        for k in targets:
+            taskdata.add_provider(localdata, self.status, k)
+            runlist.append([k, "do_%s" % task])
+        taskdata.add_unresolved(localdata, self.status)
 
-        bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
-        (filelist, masked) = self.collect_bbfiles()
-        bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
-        self.parse_bbfiles(filelist, masked, self.myProgressCallback)
-        bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
+        rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 
-        self.buildDepgraph()
+        self.server.register_idle_function(buildTargetsIdle, rq)
 
-    def cook(self):
-        """
-        We are building stuff here. We do the building
-        from here. By default we try to execute task
-        build.
-        """
+    def updateCache(self):
 
-        # Wipe the OS environment
-        bb.utils.empty_environment()
+        if self.cookerState == cookerParsed:
+            return
 
-        if self.configuration.show_environment:
-            self.showEnvironment(self.configuration.buildfile, self.configuration.pkgs_to_build)
-            sys.exit( 0 )
+        if self.cookerState != cookerParsing:
 
-        self.buildSetVars()
+            self.parseConfiguration ()
 
-        if self.configuration.interactive:
-            self.interactiveMode()
+            # Import Psyco if available and not disabled
+            import platform
+            if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
+                if not self.configuration.disable_psyco:
+                    try:
+                        import psyco
+                    except ImportError:
+                        bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
+                    else:
+                        psyco.bind( CookerParser.parse_next )
+                else:
+                    bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
 
-        if self.configuration.buildfile is not None:
-            if not self.buildFile(self.configuration.buildfile):
-                sys.exit(1)
-            sys.exit(0)
+            self.status = bb.cache.CacheData()
 
-        # initialise the parsing status now we know we will need deps
-        self.updateCache()
+            ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
+            self.status.ignored_dependencies = set(ignore.split())
+    
+            for dep in self.configuration.extra_assume_provided:
+                self.status.ignored_dependencies.add(dep)
+    
+            self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 
-        if self.configuration.revisions_changed:
-            sys.exit(bb.fetch.fetcher_compare_revisons(self.configuration.data))
+            bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
+            (filelist, masked) = self.collect_bbfiles()
+            bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
 
-        if self.configuration.parse_only:
-            bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
-            return 0
+            self.parser = CookerParser(self, filelist, masked)
+            self.cookerState = cookerParsing
 
-        pkgs_to_build = self.configuration.pkgs_to_build
+        if not self.parser.parse_next():
+            bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
+            self.buildDepgraph()
+            self.cookerState = cookerParsed
+            return None
 
-        if len(pkgs_to_build) == 0 and not self.configuration.show_versions:
-                print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
-                print "for usage information."
-                sys.exit(0)
+        return True
 
-        try:
-            if self.configuration.show_versions:
-                self.showVersions()
-                sys.exit( 0 )
-            if 'world' in pkgs_to_build:
-                self.buildWorldTargetList()
-                pkgs_to_build.remove('world')
-                for t in self.status.world_target:
-                    pkgs_to_build.append(t)
+    def checkPackages(self, pkgs_to_build):
 
-            if self.configuration.dot_graph:
-                self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
-                sys.exit( 0 )
+        if len(pkgs_to_build) == 0:
+            raise NothingToBuild
 
-            return self.buildTargets(pkgs_to_build)
+        if 'world' in pkgs_to_build:
+            self.buildWorldTargetList()
+            pkgs_to_build.remove('world')
+            for t in self.status.world_target:
+                pkgs_to_build.append(t)
 
-        except KeyboardInterrupt:
-            bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
-            sys.exit(1)
+        return pkgs_to_build
 
     def get_bbfiles( self, path = os.getcwd() ):
         """Get list of default .bb files by reading out the current directory"""
@@ -717,59 +871,108 @@ class BBCooker:
 
         return (finalfiles, masked)
 
-    def parse_bbfiles(self, filelist, masked, progressCallback = None):
-        parsed, cached, skipped, error = 0, 0, 0, 0
-        for i in xrange( len( filelist ) ):
-            f = filelist[i]
+    def serve(self):
 
-            #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
+        # Empty the environment. The environment will be populated as
+        # necessary from the data store.
+        bb.utils.empty_environment()
 
-            # read a file's metadata
+        if self.configuration.profile:
             try:
-                fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status)
-                if skip:
-                    skipped += 1
-                    bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
-                    self.bb_cache.skip(f)
-                    continue
-                elif fromCache: cached += 1
-                else: parsed += 1
-
-                # Disabled by RP as was no longer functional
-                # allow metadata files to add items to BBFILES
-                #data.update_data(self.pkgdata[f])
-                #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
-                #if addbbfiles:
-                #    for aof in addbbfiles.split():
-                #        if not files.count(aof):
-                #            if not os.path.isabs(aof):
-                #                aof = os.path.join(os.path.dirname(f),aof)
-                #            files.append(aof)
-
-                # now inform the caller
-                if progressCallback is not None:
-                    progressCallback( i + 1, len( filelist ), f, fromCache )
+                import cProfile as profile
+            except:
+                import profile
+
+            profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
+
+            # Redirect stdout to capture profile information
+            pout = open('profile.log.processed', 'w')
+            so = sys.stdout.fileno()
+            os.dup2(pout.fileno(), so)
+
+            import pstats
+            p = pstats.Stats('profile.log')
+            p.sort_stats('time')
+            p.print_stats()
+            p.print_callers()
+            p.sort_stats('cumulative')
+            p.print_stats()
+
+            os.dup2(so, pout.fileno())
+            pout.flush()
+            pout.close()
+        else:
+            self.server.serve_forever()
+        
+        bb.event.fire(CookerExit(), self.configuration.event_data)
+        
+class CookerExit(bb.event.Event):
+    """
+    Notify clients of the Cooker shutdown
+    """
+
+    def __init__(self):
+        bb.event.Event.__init__(self)
+
+class CookerParser:
+    def __init__(self, cooker, filelist, masked):
+        # Internal data
+        self.filelist = filelist
+        self.cooker = cooker
+
+        # Accounting statistics
+        self.parsed = 0
+        self.cached = 0
+        self.error = 0
+        self.masked = masked
+        self.total = len(filelist)
+
+        self.skipped = 0
+        self.virtuals = 0
+
+        # Pointer to the next file to parse
+        self.pointer = 0
+
+    def parse_next(self):
+        if self.pointer < len(self.filelist):
+            f = self.filelist[self.pointer]
+            cooker = self.cooker
+
+            try:
+                fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
+                if fromCache:
+                    self.cached += 1
+                else:
+                    self.parsed += 1
+
+                self.skipped += skipped
+                self.virtuals += virtuals
 
             except IOError, e:
-                self.bb_cache.remove(f)
+                self.error += 1
+                cooker.bb_cache.remove(f)
                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
                 pass
             except KeyboardInterrupt:
-                self.bb_cache.sync()
+                cooker.bb_cache.remove(f)
+                cooker.bb_cache.sync()
                 raise
             except Exception, e:
-                error += 1
-                self.bb_cache.remove(f)
+                self.error += 1
+                cooker.bb_cache.remove(f)
                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
             except:
-                self.bb_cache.remove(f)
+                cooker.bb_cache.remove(f)
                 raise
+            finally:
+                bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data)
 
-        if progressCallback is not None:
-            print "\r" # need newline after Handling Bitbake files message
-            bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
+            self.pointer += 1
 
-        self.bb_cache.sync()
+        if self.pointer >= self.total:
+            cooker.bb_cache.sync()
+            if self.error > 0:
+                raise ParsingErrorsFound
+            return False
+        return True
 
-        if error > 0:
-            bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")