9 files changed, 2350 insertions, 0 deletions

diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
new file mode 100755
index 0000000000..88893f8de6
--- /dev/null
+++ b/bitbake/bin/bitbake
@@ -0,0 +1,334 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003, 2004  Chris Larson
+# Copyright (C) 2003, 2004  Phil Blundell
+# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
+# Copyright (C) 2005        Holger Hans Peter Freyther
+# Copyright (C) 2005        ROAD GmbH
+# Copyright (C) 2006        Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys, logging
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),
+                                'lib'))
+
+import optparse
+import warnings
+from traceback import format_exception
+try:
+    import bb
+except RuntimeError as exc:
+    sys.exit(str(exc))
+from bb import event
+import bb.msg
+from bb import cooker
+from bb import ui
+from bb import server
+from bb import cookerdata
+
+__version__ = "1.20.0"
+logger = logging.getLogger("BitBake")
+
+# Python multiprocessing requires /dev/shm
+if not os.access('/dev/shm', os.W_OK | os.X_OK):
+    sys.exit("FATAL: /dev/shm does not exist or is not writable")
+
+# Unbuffer stdout to avoid log truncation in the event
+# of an unorderly exit as well as to provide timely
+# updates to log files for use with tail
+try:
+    if sys.stdout.name == '<stdout>':
+        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+except:
+    pass
+
+
+def get_ui(config):
+    if not config.ui:
+        # modify 'ui' attribute because it is also read by cooker
+        config.ui = os.environ.get('BITBAKE_UI', 'knotty')
+
+    interface = config.ui
+
+    try:
+        # Dynamically load the UI based on the ui name. Although we
+        # suggest a fixed set this allows you to have flexibility in which
+        # ones are available.
+        module = __import__("bb.ui", fromlist = [interface])
+        return getattr(module, interface)
+    except AttributeError:
+        sys.exit("FATAL: Invalid user interface '%s' specified.\n"
+                 "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
+
+
+# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
+warnlog = logging.getLogger("BitBake.Warnings")
+_warnings_showwarning = warnings.showwarning
+def _showwarning(message, category, filename, lineno, file=None, line=None):
+    if file is not None:
+        if _warnings_showwarning is not None:
+            _warnings_showwarning(message, category, filename, lineno, file, line)
+    else:
+        s = warnings.formatwarning(message, category, filename, lineno)
+        warnlog.warn(s)
+
+warnings.showwarning = _showwarning
+warnings.filterwarnings("ignore")
+warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)")
+warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
+warnings.filterwarnings("ignore", category=ImportWarning)
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
+warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
+
+class BitBakeConfigParameters(cookerdata.ConfigParameters):
+
+    def parseCommandLine(self):
+        parser = optparse.OptionParser(
+            version = "BitBake Build Tool Core version %s, %%prog version %s" % (bb.__version__, __version__),
+            usage = """%prog [options] [recipename/target ...]
+
+    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
+    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+    will provide the layer, BBFILES and other configuration information.""")
+
+        parser.add_option("-b", "--buildfile", help = "Execute tasks from a specific .bb recipe directly. WARNING: Does not handle any dependencies from other recipes.",
+                   action = "store", dest = "buildfile", default = None)
+
+        parser.add_option("-k", "--continue", help = "Continue as much as possible after an error. While the target that failed and anything depending on it cannot be built, as much as possible will be built before stopping.",
+                   action = "store_false", dest = "abort", default = True)
+
+        parser.add_option("-a", "--tryaltconfigs", help = "Continue with builds by trying to use alternative providers where possible.",
+                   action = "store_true", dest = "tryaltconfigs", default = False)
+
+        parser.add_option("-f", "--force", help = "Force the specified targets/task to run (invalidating any existing stamp file).",
+                   action = "store_true", dest = "force", default = False)
+
+        parser.add_option("-c", "--cmd", help = "Specify the task to execute. The exact options available depend on the metadata. Some examples might be 'compile' or 'populate_sysroot' or 'listtasks' may give a list of the tasks available.",
+                   action = "store", dest = "cmd")
+
+        parser.add_option("-C", "--clear-stamp", help = "Invalidate the stamp for the specified task such as 'compile' and then run the default task for the specified target(s).",
+                    action = "store", dest = "invalidate_stamp")
+
+        parser.add_option("-r", "--read", help = "Read the specified file before bitbake.conf.",
+                   action = "append", dest = "prefile", default = [])
+
+        parser.add_option("-R", "--postread", help = "Read the specified file after bitbake.conf.",
+                          action = "append", dest = "postfile", default = [])
+
+        parser.add_option("-v", "--verbose", help = "Output more log message data to the terminal.",
+                   action = "store_true", dest = "verbose", default = False)
+
+        parser.add_option("-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
+                   action = "count", dest="debug", default = 0)
+
+        parser.add_option("-n", "--dry-run", help = "Don't execute, just go through the motions.",
+                   action = "store_true", dest = "dry_run", default = False)
+
+        parser.add_option("-S", "--dump-signatures", help = "Don't execute, just dump out the signature construction information.",
+                   action = "store_true", dest = "dump_signatures", default = False)
+
+        parser.add_option("-p", "--parse-only", help = "Quit after parsing the BB recipes.",
+                   action = "store_true", dest = "parse_only", default = False)
+
+        parser.add_option("-s", "--show-versions", help = "Show current and preferred versions of all recipes.",
+                   action = "store_true", dest = "show_versions", default = False)
+
+        parser.add_option("-e", "--environment", help = "Show the global or per-package environment complete with information about where variables were set/changed.",
+                   action = "store_true", dest = "show_environment", default = False)
+
+        parser.add_option("-g", "--graphviz", help = "Save dependency tree information for the specified targets in the dot syntax.",
+                    action = "store_true", dest = "dot_graph", default = False)
+
+        parser.add_option("-I", "--ignore-deps", help = """Assume these dependencies don't exist and are already provided (equivalent to ASSUME_PROVIDED). Useful to make dependency graphs more appealing""",
+                    action = "append", dest = "extra_assume_provided", default = [])
+
+        parser.add_option("-l", "--log-domains", help = """Show debug logging for the specified logging domains""",
+                    action = "append", dest = "debug_domains", default = [])
+
+        parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
+                   action = "store_true", dest = "profile", default = False)
+
+        parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
+                   action = "store", dest = "ui")
+
+        parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
+                   action = "store", dest = "servertype")
+
+        parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not.",
+                   action = "store_true", dest = "revisions_changed", default = False)
+
+        parser.add_option("", "--server-only", help = "Run bitbake without a UI, only starting a server (cooker) process.",
+                   action = "store_true", dest = "server_only", default = False)
+
+        parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to.",
+                   action = "store", dest = "bind", default = False)
+
+        parser.add_option("", "--no-setscene", help = "Do not run any setscene tasks. sstate will be ignored and everything needed, built.",
+                   action = "store_true", dest = "nosetscene", default = False)
+
+        parser.add_option("", "--remote-server", help = "Connect to the specified server.",
+                   action = "store", dest = "remote_server", default = False)
+
+        parser.add_option("-m", "--kill-server", help = "Terminate the remote server.",
+                    action = "store_true", dest = "kill_server", default = False)
+
+        parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client.",
+                   action = "store_true", dest = "observe_only", default = False)
+
+        options, targets = parser.parse_args(sys.argv)
+
+        # some environmental variables set also configuration options
+        if "BBSERVER" in os.environ:
+            options.servertype = "xmlrpc"
+            options.remote_server = os.environ["BBSERVER"]
+
+        return options, targets[1:]
+
+
+def start_server(servermodule, configParams, configuration):
+    server = servermodule.BitBakeServer()
+    if configParams.bind:
+        (host, port) = configParams.bind.split(':')
+        server.initServer((host, int(port)))
+    else:
+        server.initServer()
+
+    try:
+        configuration.setServerRegIdleCallback(server.getServerIdleCB())
+
+        cooker = bb.cooker.BBCooker(configuration)
+
+        server.addcooker(cooker)
+        server.saveConnectionDetails()
+    except Exception as e:
+        exc_info = sys.exc_info()
+        while True:
+            try:
+                import queue
+            except ImportError:
+                import Queue as queue
+            try:
+                event = server.event_queue.get(block=False)
+            except (queue.Empty, IOError):
+                break
+            if isinstance(event, logging.LogRecord):
+                logger.handle(event)
+        raise exc_info[1], None, exc_info[2]
+    server.detach()
+    return server
+
+
+
+def main():
+
+    configParams = BitBakeConfigParameters()
+    configuration = cookerdata.CookerConfiguration()
+    configuration.setConfigParameters(configParams)
+
+    ui_module = get_ui(configParams)
+
+    # Server type can be xmlrpc or process currently, if nothing is specified,
+    # the default server is process
+    if configParams.servertype:
+        server_type = configParams.servertype
+    else:
+        server_type = 'process'
+
+    try:
+        module = __import__("bb.server", fromlist = [server_type])
+        servermodule = getattr(module, server_type)
+    except AttributeError:
+        sys.exit("FATAL: Invalid server type '%s' specified.\n"
+                 "Valid interfaces: xmlrpc, process [default]." % servertype)
+
+    if configParams.server_only:
+        if configParams.servertype != "xmlrpc":
+            sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+        if not configParams.bind:
+            sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n")
+        if configParams.remote_server:
+            sys.exit("FATAL: The '--server-only' option conflicts with %s.\n" % 
+                ("the BBSERVER environment variable" if "BBSERVER" in os.environ else "the '--remote-server' option" ))
+
+    if configParams.bind and configParams.servertype != "xmlrpc":
+        sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n")
+
+    if configParams.remote_server and configParams.servertype != "xmlrpc":
+        sys.exit("FATAL: If '--remote-server' is defined, we must set the servertype as 'xmlrpc'.\n")
+
+    if configParams.observe_only and (not configParams.remote_server or configParams.bind):
+        sys.exit("FATAL: '--observe-only' can only be used by UI clients connecting to a server.\n")
+
+    if "BBDEBUG" in os.environ:
+        level = int(os.environ["BBDEBUG"])
+        if level > configuration.debug:
+            configuration.debug = level
+
+    bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
+                         configuration.debug_domains)
+
+    # Ensure logging messages get sent to the UI as events
+    handler = bb.event.LogHandler()
+    logger.addHandler(handler)
+
+    # Clear away any spurious environment variables while we stoke up the cooker
+    cleanedvars = bb.utils.clean_environment()
+
+    if not configParams.remote_server:
+        # we start a server with a given configuration
+        server = start_server(servermodule, configParams, configuration)
+        bb.event.ui_queue = []
+    else:
+        # we start a stub server that is actually a XMLRPClient that connects to a real server
+        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
+        server.saveConnectionDetails(configParams.remote_server)
+
+    if not configParams.server_only:
+        # Collect the feature set for the UI
+        featureset = getattr(ui_module, "featureSet", [])
+
+        # Setup a connection to the server (cooker)
+        server_connection = server.establishConnection(featureset)
+
+        # Restore the environment in case the UI needs it
+        for k in cleanedvars:
+            os.environ[k] = cleanedvars[k]
+
+        logger.removeHandler(handler)
+
+        try:
+            return ui_module.main(server_connection.connection, server_connection.events, configParams)
+        finally:
+            bb.event.ui_queue = []
+            server_connection.terminate()
+    else:
+        print("server address: %s, server port: %s" % (server.serverImpl.host, server.serverImpl.port))
+
+    return 1
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except bb.BBHandledException:
+        ret = 1
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)
+
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
new file mode 100755
index 0000000000..78757b0aae
--- /dev/null
+++ b/bitbake/bin/bitbake-diffsigs
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+# bitbake-diffsigs
+# BitBake task signature data comparison utility
+#
+# Copyright (C) 2012-2013 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+import warnings
+import fnmatch
+import optparse
+import logging
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+
+import bb.tinfoil
+import bb.siggen
+
+def logger_create(name, output=sys.stderr):
+    logger = logging.getLogger(name)
+    console = logging.StreamHandler(output)
+    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+    if output.isatty():
+        format.enable_color()
+    console.setFormatter(format)
+    logger.addHandler(console)
+    logger.setLevel(logging.INFO)
+    return logger
+
+logger = logger_create('bitbake-diffsigs')
+
+def find_compare_task(bbhandler, pn, taskname):
+    """ Find the most recent signature files for the specified PN/task and compare them """
+
+    if not hasattr(bb.siggen, 'find_siginfo'):
+        logger.error('Metadata does not support finding signature data files')
+        sys.exit(1)
+
+    if not taskname.startswith('do_'):
+        taskname = 'do_%s' % taskname
+
+    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
+    if not latestfiles:
+        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+        sys.exit(1)
+    elif len(latestfiles) < 2:
+        logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
+        sys.exit(1)
+    else:
+        # Define recursion callback
+        def recursecb(key, hash1, hash2):
+            hashes = [hash1, hash2]
+            hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
+
+            recout = []
+            if len(hashfiles) == 2:
+                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+                recout.extend(list('  ' + l for l in out2))
+            else:
+                recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+
+            return recout
+
+        # Recurse into signature comparison
+        output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb)
+        if output:
+            print '\n'.join(output)
+    sys.exit(0)
+
+
+
+parser = optparse.OptionParser(
+    description = "Compares siginfo/sigdata files written out by BitBake",
+    usage = """
+  %prog -t recipename taskname
+  %prog sigdatafile1 sigdatafile2
+  %prog sigdatafile1""")
+
+parser.add_option("-t", "--task",
+        help = "find the signature data files for last two runs of the specified task and compare them",
+        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
+
+options, args = parser.parse_args(sys.argv)
+
+if options.taskargs:
+    tinfoil = bb.tinfoil.Tinfoil()
+    tinfoil.prepare(config_only = True)
+    find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1])
+else:
+    if len(args) == 1:
+        parser.print_help()
+    else:
+        import cPickle
+        try:
+            if len(args) == 2:
+                output = bb.siggen.dump_sigfile(sys.argv[1])
+            else:
+                output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
+        except IOError as e:
+            logger.error(str(e))
+            sys.exit(1)
+        except cPickle.UnpicklingError, EOFError:
+            logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
+            sys.exit(1)
+
+    if output:
+        print '\n'.join(output)
diff --git a/bitbake/bin/bitbake-dumpsig b/bitbake/bin/bitbake-dumpsig
new file mode 100755
index 0000000000..656d93a5ac
--- /dev/null
+++ b/bitbake/bin/bitbake-dumpsig
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+# bitbake-dumpsig
+# BitBake task signature dump utility
+#
+# Copyright (C) 2013 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+import warnings
+import optparse
+import logging
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+
+import bb.siggen
+
+def logger_create(name, output=sys.stderr):
+    logger = logging.getLogger(name)
+    console = logging.StreamHandler(output)
+    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+    if output.isatty():
+        format.enable_color()
+    console.setFormatter(format)
+    logger.addHandler(console)
+    logger.setLevel(logging.INFO)
+    return logger
+
+logger = logger_create('bitbake-dumpsig')
+
+parser = optparse.OptionParser(
+    description = "Dumps siginfo/sigdata files written out by BitBake",
+    usage = """
+  %prog sigdatafile""")
+
+options, args = parser.parse_args(sys.argv)
+
+if len(args) == 1:
+    parser.print_help()
+else:
+    import cPickle
+    try:
+        output = bb.siggen.dump_sigfile(args[1])
+    except IOError as e:
+        logger.error(str(e))
+        sys.exit(1)
+    except cPickle.UnpicklingError, EOFError:
+        logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
+        sys.exit(1)
+
+    if output:
+        print '\n'.join(output)
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
new file mode 100755
index 0000000000..047583c497
--- /dev/null
+++ b/bitbake/bin/bitbake-layers
@@ -0,0 +1,720 @@
+#!/usr/bin/env python
+
+# This script has subcommands which operate against your bitbake layers, either
+# displaying useful information, or acting against them.
+# See the help output for details on available commands.
+
+# Copyright (C) 2011 Mentor Graphics Corporation
+# Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cmd
+import logging
+import os
+import sys
+import fnmatch
+from collections import defaultdict
+import re
+
+bindir = os.path.dirname(__file__)
+topdir = os.path.dirname(bindir)
+sys.path[0:0] = [os.path.join(topdir, 'lib')]
+
+import bb.cache
+import bb.cooker
+import bb.providers
+import bb.utils
+import bb.tinfoil
+
+
+logger = logging.getLogger('BitBake')
+
+
+def main(args):
+    cmds = Commands()
+    if args:
+        # Allow user to specify e.g. show-layers instead of show_layers
+        args = [args[0].replace('-', '_')] + args[1:]
+        cmds.onecmd(' '.join(args))
+    else:
+        cmds.do_help('')
+    return cmds.returncode
+
+
+class Commands(cmd.Cmd):
+    def __init__(self):
+        cmd.Cmd.__init__(self)
+        self.bbhandler = bb.tinfoil.Tinfoil()
+        self.returncode = 0
+        self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
+
+    def default(self, line):
+        """Handle unrecognised commands"""
+        sys.stderr.write("Unrecognised command or option\n")
+        self.do_help('')
+
+    def do_help(self, topic):
+        """display general help or help on a specified command"""
+        if topic:
+            sys.stdout.write('%s: ' % topic)
+            cmd.Cmd.do_help(self, topic.replace('-', '_'))
+        else:
+            sys.stdout.write("usage: bitbake-layers <command> [arguments]\n\n")
+            sys.stdout.write("Available commands:\n")
+            procnames = list(set(self.get_names()))
+            for procname in procnames:
+                if procname[:3] == 'do_':
+                    sys.stdout.write("  %s\n" % procname[3:].replace('_', '-'))
+                    doc = getattr(self, procname).__doc__
+                    if doc:
+                        sys.stdout.write("    %s\n" % doc.splitlines()[0])
+
+    def do_show_layers(self, args):
+        """show current configured layers"""
+        self.bbhandler.prepare(config_only = True)
+        logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
+        logger.plain('=' * 74)
+        for layerdir in self.bblayers:
+            layername = self.get_layer_name(layerdir)
+            layerpri = 0
+            for layer, _, regex, pri in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
+                if regex.match(os.path.join(layerdir, 'test')):
+                    layerpri = pri
+                    break
+
+            logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
+
+
+    def version_str(self, pe, pv, pr = None):
+        verstr = "%s" % pv
+        if pr:
+            verstr = "%s-%s" % (verstr, pr)
+        if pe:
+            verstr = "%s:%s" % (pe, verstr)
+        return verstr
+
+
+    def do_show_overlayed(self, args):
+        """list overlayed recipes (where the same recipe exists in another layer)
+
+usage: show-overlayed [-f] [-s]
+
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Note that skipped recipes that
+are overlayed will also be listed, with a " (skipped)" suffix.
+
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with the ones they overlay indented underneath
+  -s   only list overlayed recipes where the version is the same
+"""
+        self.bbhandler.prepare()
+
+        show_filenames = False
+        show_same_ver_only = False
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-s':
+                show_same_ver_only = True
+            else:
+                sys.stderr.write("show-overlayed: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+
+        items_listed = self.list_recipes('Overlayed recipes', None, True, show_same_ver_only, show_filenames, True)
+
+        # Check for overlayed .bbclass files
+        classes = defaultdict(list)
+        for layerdir in self.bblayers:
+            classdir = os.path.join(layerdir, 'classes')
+            if os.path.exists(classdir):
+                for classfile in os.listdir(classdir):
+                    if os.path.splitext(classfile)[1] == '.bbclass':
+                        classes[classfile].append(classdir)
+
+        # Locating classes and other files is a bit more complicated than recipes -
+        # layer priority is not a factor; instead BitBake uses the first matching
+        # file in BBPATH, which is manipulated directly by each layer's
+        # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
+        # factor - however, each layer.conf is free to either prepend or append to
+        # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
+        # not be exactly the order present in bblayers.conf either.
+        bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
+        overlayed_class_found = False
+        for (classfile, classdirs) in classes.items():
+            if len(classdirs) > 1:
+                if not overlayed_class_found:
+                    logger.plain('=== Overlayed classes ===')
+                    overlayed_class_found = True
+
+                mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
+                if show_filenames:
+                    logger.plain('%s' % mainfile)
+                else:
+                    # We effectively have to guess the layer here
+                    logger.plain('%s:' % classfile)
+                    mainlayername = '?'
+                    for layerdir in self.bblayers:
+                        classdir = os.path.join(layerdir, 'classes')
+                        if mainfile.startswith(classdir):
+                            mainlayername = self.get_layer_name(layerdir)
+                    logger.plain('  %s' % mainlayername)
+                for classdir in classdirs:
+                    fullpath = os.path.join(classdir, classfile)
+                    if fullpath != mainfile:
+                        if show_filenames:
+                            print('  %s' % fullpath)
+                        else:
+                            print('  %s' % self.get_layer_name(os.path.dirname(classdir)))
+
+        if overlayed_class_found:
+            items_listed = True;
+
+        if not items_listed:
+            logger.plain('No overlayed files found.')
+
+
+    def do_show_recipes(self, args):
+        """list available recipes, showing the layer they are provided by
+
+usage: show-recipes [-f] [-m] [pnspec]
+
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Optionally you may specify
+pnspec to match a specified recipe name (supports wildcards). Note that
+skipped recipes will also be listed, with a " (skipped)" suffix.
+
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with other available recipes indented underneath
+  -m   only list where multiple recipes (in the same layer or different
+       layers) exist for the same recipe name
+"""
+        self.bbhandler.prepare()
+
+        show_filenames = False
+        show_multi_provider_only = False
+        pnspec = None
+        title = 'Available recipes:'
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-m':
+                show_multi_provider_only = True
+            elif not arg.startswith('-'):
+                pnspec = arg
+                title = 'Available recipes matching %s:' % pnspec
+            else:
+                sys.stderr.write("show-recipes: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+        self.list_recipes(title, pnspec, False, False, show_filenames, show_multi_provider_only)
+
+
+    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only):
+        pkg_pn = self.bbhandler.cooker.recipecache.pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.config_data, self.bbhandler.cooker.recipecache, pkg_pn)
+        allproviders = bb.providers.allProviders(self.bbhandler.cooker.recipecache)
+
+        # Ensure we list skipped recipes
+        # We are largely guessing about PN, PV and the preferred version here,
+        # but we have no choice since skipped recipes are not fully parsed
+        skiplist = self.bbhandler.cooker.skiplist.keys()
+        skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.collection.calc_bbfile_priority(fileitem) )
+        skiplist.reverse()
+        for fn in skiplist:
+            recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
+            p = recipe_parts[0]
+            if len(recipe_parts) > 1:
+                ver = (None, recipe_parts[1], None)
+            else:
+                ver = (None, 'unknown', None)
+            allproviders[p].append((ver, fn))
+            if not p in pkg_pn:
+                pkg_pn[p] = 'dummy'
+                preferred_versions[p] = (ver, fn)
+
+        def print_item(f, pn, ver, layer, ispref):
+            if f in skiplist:
+                skipped = ' (skipped)'
+            else:
+                skipped = ''
+            if show_filenames:
+                if ispref:
+                    logger.plain("%s%s", f, skipped)
+                else:
+                    logger.plain("  %s%s", f, skipped)
+            else:
+                if ispref:
+                    logger.plain("%s:", pn)
+                logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
+
+        preffiles = []
+        items_listed = False
+        for p in sorted(pkg_pn):
+            if pnspec:
+                if not fnmatch.fnmatch(p, pnspec):
+                    continue
+
+            if len(allproviders[p]) > 1 or not show_multi_provider_only:
+                pref = preferred_versions[p]
+                preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0]
+                if preffile not in preffiles:
+                    preflayer = self.get_file_layer(preffile)
+                    multilayer = False
+                    same_ver = True
+                    provs = []
+                    for prov in allproviders[p]:
+                        provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
+                        provlayer = self.get_file_layer(provfile)
+                        provs.append((provfile, provlayer, prov[0]))
+                        if provlayer != preflayer:
+                            multilayer = True
+                        if prov[0] != pref[0]:
+                            same_ver = False
+
+                    if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
+                        if not items_listed:
+                            logger.plain('=== %s ===' % title)
+                            items_listed = True
+                        print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
+                        for (provfile, provlayer, provver) in provs:
+                            if provfile != preffile:
+                                print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
+                        # Ensure we don't show two entries for BBCLASSEXTENDed recipes
+                        preffiles.append(preffile)
+
+        return items_listed
+
+
+    def do_flatten(self, args):
+        """flattens layer configuration into a separate output directory.
+
+usage: flatten [layer1 layer2 [layer3]...] <outputdir>
+
+Takes the specified layers (or all layers in the current layer
+configuration if none are specified) and builds a "flattened" directory
+containing the contents of all layers, with any overlayed recipes removed
+and bbappends appended to the corresponding recipes. Note that some manual
+cleanup may still be necessary afterwards, in particular:
+
+* where non-recipe files (such as patches) are overwritten (the flatten
+  command will show a warning for these)
+* where anything beyond the normal layer setup has been added to
+  layer.conf (only the lowest priority number layer's layer.conf is used)
+* overridden/appended items from bbappends will need to be tidied up
+* when the flattened layers do not have the same directory structure (the
+  flatten command should show a warning when this will cause a problem)
+
+Warning: if you flatten several layers where another layer is intended to
+be used "inbetween" them (in layer priority order) such that recipes /
+bbappends in the layers interact, and then attempt to use the new output
+layer together with that other layer, you may no longer get the same
+build results (as the layer priority order has effectively changed).
+"""
+        arglist = args.split()
+        if len(arglist) < 1:
+            logger.error('Please specify an output directory')
+            self.do_help('flatten')
+            return
+
+        if len(arglist) == 2:
+            logger.error('If you specify layers to flatten you must specify at least two')
+            self.do_help('flatten')
+            return
+
+        outputdir = arglist[-1]
+        if os.path.exists(outputdir) and os.listdir(outputdir):
+            logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
+            return
+
+        self.bbhandler.prepare()
+        layers = self.bblayers
+        if len(arglist) > 2:
+            layernames = arglist[:-1]
+            found_layernames = []
+            found_layerdirs = []
+            for layerdir in layers:
+                layername = self.get_layer_name(layerdir)
+                if layername in layernames:
+                    found_layerdirs.append(layerdir)
+                    found_layernames.append(layername)
+
+            for layername in layernames:
+                if not layername in found_layernames:
+                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
+                    return
+            layers = found_layerdirs
+        else:
+            layernames = []
+
+        # Ensure a specified path matches our list of layers
+        def layer_path_match(path):
+            for layerdir in layers:
+                if path.startswith(os.path.join(layerdir, '')):
+                    return layerdir
+            return None
+
+        appended_recipes = []
+        for layer in layers:
+            overlayed = []
+            for f in self.bbhandler.cooker.collection.overlayed.iterkeys():
+                for of in self.bbhandler.cooker.collection.overlayed[f]:
+                    if of.startswith(layer):
+                        overlayed.append(of)
+
+            logger.plain('Copying files from %s...' % layer )
+            for root, dirs, files in os.walk(layer):
+                for f1 in files:
+                    f1full = os.sep.join([root, f1])
+                    if f1full in overlayed:
+                        logger.plain('  Skipping overlayed file %s' % f1full )
+                    else:
+                        ext = os.path.splitext(f1)[1]
+                        if ext != '.bbappend':
+                            fdest = f1full[len(layer):]
+                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
+                            bb.utils.mkdirhier(os.path.dirname(fdest))
+                            if os.path.exists(fdest):
+                                if f1 == 'layer.conf' and root.endswith('/conf'):
+                                    logger.plain('  Skipping layer config file %s' % f1full )
+                                    continue
+                                else:
+                                    logger.warn('Overwriting file %s', fdest)
+                            bb.utils.copyfile(f1full, fdest)
+                            if ext == '.bb':
+                                if f1 in self.bbhandler.cooker.collection.appendlist:
+                                    appends = self.bbhandler.cooker.collection.appendlist[f1]
+                                    if appends:
+                                        logger.plain('  Applying appends to %s' % fdest )
+                                        for appendname in appends:
+                                            if layer_path_match(appendname):
+                                                self.apply_append(appendname, fdest)
+                                    appended_recipes.append(f1)
+
+        # Take care of when some layers are excluded and yet we have included bbappends for those recipes
+        for recipename in self.bbhandler.cooker.collection.appendlist.iterkeys():
+            if recipename not in appended_recipes:
+                appends = self.bbhandler.cooker.collection.appendlist[recipename]
+                first_append = None
+                for appendname in appends:
+                    layer = layer_path_match(appendname)
+                    if layer:
+                        if first_append:
+                            self.apply_append(appendname, first_append)
+                        else:
+                            fdest = appendname[len(layer):]
+                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
+                            bb.utils.mkdirhier(os.path.dirname(fdest))
+                            bb.utils.copyfile(appendname, fdest)
+                            first_append = fdest
+
+        # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
+        # have come from)
+        first_regex = None
+        layerdir = layers[0]
+        for layername, pattern, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
+            if regex.match(os.path.join(layerdir, 'test')):
+                first_regex = regex
+                break
+
+        if first_regex:
+            # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
+            bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split()
+            bbfiles_layer = []
+            for item in bbfiles:
+                if first_regex.match(item):
+                    newpath = os.path.join(outputdir, item[len(layerdir)+1:])
+                    bbfiles_layer.append(newpath)
+
+            if bbfiles_layer:
+                # Check that all important layer files match BBFILES
+                for root, dirs, files in os.walk(outputdir):
+                    for f1 in files:
+                        ext = os.path.splitext(f1)[1]
+                        if ext in ['.bb', '.bbappend']:
+                            f1full = os.sep.join([root, f1])
+                            entry_found = False
+                            for item in bbfiles_layer:
+                                if fnmatch.fnmatch(f1full, item):
+                                    entry_found = True
+                                    break
+                            if not entry_found:
+                                logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
+
+    def get_file_layer(self, filename):
+        for layer, _, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
+            if regex.match(filename):
+                for layerdir in self.bblayers:
+                    if regex.match(os.path.join(layerdir, 'test')) and re.match(layerdir, filename):
+                        return self.get_layer_name(layerdir)
+        return "?"
+
+    def get_file_layerdir(self, filename):
+        for layer, _, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
+            if regex.match(filename):
+                for layerdir in self.bblayers:
+                    if regex.match(os.path.join(layerdir, 'test')) and re.match(layerdir, filename):
+                        return layerdir
+        return "?"
+
+    def remove_layer_prefix(self, f):
+        """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the
+           return value will be: layer_dir/foo/blah"""
+        f_layerdir = self.get_file_layerdir(f)
+        prefix = os.path.join(os.path.dirname(f_layerdir), '')
+        return f[len(prefix):] if f.startswith(prefix) else f
+
+    def get_layer_name(self, layerdir):
+        return os.path.basename(layerdir.rstrip(os.sep))
+
+    def apply_append(self, appendname, recipename):
+        appendfile = open(appendname, 'r')
+        recipefile = open(recipename, 'a')
+        recipefile.write('\n')
+        recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
+        recipefile.writelines(appendfile.readlines())
+        recipefile.close()
+        appendfile.close()
+
+    def do_show_appends(self, args):
+        """list bbappend files and recipe files they apply to
+
+usage: show-appends
+
+Recipes are listed with the bbappends that apply to them as subitems.
+"""
+        self.bbhandler.prepare()
+        if not self.bbhandler.cooker.collection.appendlist:
+            logger.plain('No append files found')
+            return
+
+        logger.plain('=== Appended recipes ===')
+
+        pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
+        pnlist.sort()
+        for pn in pnlist:
+            self.show_appends_for_pn(pn)
+
+        self.show_appends_for_skipped()
+
+    def show_appends_for_pn(self, pn):
+        filenames = self.bbhandler.cooker_data.pkg_pn[pn]
+
+        best = bb.providers.findBestProvider(pn,
+                                             self.bbhandler.config_data,
+                                             self.bbhandler.cooker_data,
+                                             self.bbhandler.cooker_data.pkg_pn)
+        best_filename = os.path.basename(best[3])
+
+        self.show_appends_output(filenames, best_filename)
+
+    def show_appends_for_skipped(self):
+        filenames = [os.path.basename(f)
+                    for f in self.bbhandler.cooker.skiplist.iterkeys()]
+        self.show_appends_output(filenames, None, " (skipped)")
+
+    def show_appends_output(self, filenames, best_filename, name_suffix = ''):
+        appended, missing = self.get_appends_for_files(filenames)
+        if appended:
+            for basename, appends in appended:
+                logger.plain('%s%s:', basename, name_suffix)
+                for append in appends:
+                    logger.plain('  %s', append)
+
+            if best_filename:
+                if best_filename in missing:
+                    logger.warn('%s: missing append for preferred version',
+                                best_filename)
+                    self.returncode |= 1
+
+
+    def get_appends_for_files(self, filenames):
+        appended, notappended = [], []
+        for filename in filenames:
+            _, cls = bb.cache.Cache.virtualfn2realfn(filename)
+            if cls:
+                continue
+
+            basename = os.path.basename(filename)
+            appends = self.bbhandler.cooker.collection.appendlist.get(basename)
+            if appends:
+                appended.append((basename, list(appends)))
+            else:
+                notappended.append(basename)
+        return appended, notappended
+
+    def do_show_cross_depends(self, args):
+        """figure out the dependency between recipes that crosses a layer boundary.
+
+usage: show-cross-depends [-f]
+
+Figure out the dependency between recipes that crosses a layer boundary.
+
+Options:
+  -f   show full file path
+
+NOTE:
+The .bbappend file can impact the dependency.
+"""
+        self.bbhandler.prepare()
+
+        show_filenames = False
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            else:
+                sys.stderr.write("show-cross-depends: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+
+        pkg_fn = self.bbhandler.cooker_data.pkg_fn
+        bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
+        self.require_re = re.compile(r"require\s+(.+)")
+        self.include_re = re.compile(r"include\s+(.+)")
+        self.inherit_re = re.compile(r"inherit\s+(.+)")
+
+        # The bb's DEPENDS and RDEPENDS
+        for f in pkg_fn:
+            f = bb.cache.Cache.virtualfn2realfn(f)[0]
+            # Get the layername that the file is in
+            layername = self.get_file_layer(f)
+
+            # The DEPENDS
+            deps = self.bbhandler.cooker_data.deps[f]
+            for pn in deps:
+                if pn in self.bbhandler.cooker_data.pkg_pn:
+                    best = bb.providers.findBestProvider(pn,
+                            self.bbhandler.config_data,
+                            self.bbhandler.cooker_data,
+                            self.bbhandler.cooker_data.pkg_pn)
+                    self.check_cross_depends("DEPENDS", layername, f, best[3], show_filenames)
+
+            # The RDPENDS
+            all_rdeps = self.bbhandler.cooker_data.rundeps[f].values()
+            # Remove the duplicated or null one.
+            sorted_rdeps = {}
+            # The all_rdeps is the list in list, so we need two for loops
+            for k1 in all_rdeps:
+                for k2 in k1:
+                    sorted_rdeps[k2] = 1
+            all_rdeps = sorted_rdeps.keys()
+            for rdep in all_rdeps:
+                all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rdep)
+                if all_p:
+                    best = bb.providers.filterProvidersRunTime(all_p, rdep,
+                                    self.bbhandler.config_data,
+                                    self.bbhandler.cooker_data)[0][0]
+                    self.check_cross_depends("RDEPENDS", layername, f, best, show_filenames)
+
+            # The inherit class
+            cls_re = re.compile('classes/')
+            if f in self.bbhandler.cooker_data.inherits:
+                inherits = self.bbhandler.cooker_data.inherits[f]
+                for cls in inherits:
+                    # The inherits' format is [classes/cls, /path/to/classes/cls]
+                    # ignore the classes/cls.
+                    if not cls_re.match(cls):
+                        inherit_layername = self.get_file_layer(cls)
+                        if inherit_layername != layername:
+                            if not show_filenames:
+                                f_short = self.remove_layer_prefix(f)
+                                cls = self.remove_layer_prefix(cls)
+                            else:
+                                f_short = f
+                            logger.plain("%s inherits %s" % (f_short, cls))
+
+            # The 'require/include xxx' in the bb file
+            pv_re = re.compile(r"\${PV}")
+            fnfile = open(f, 'r')
+            line = fnfile.readline()
+            while line:
+                m, keyword = self.match_require_include(line)
+                # Found the 'require/include xxxx'
+                if m:
+                    needed_file = m.group(1)
+                    # Replace the ${PV} with the real PV
+                    if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
+                        pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
+                        needed_file = re.sub(r"\${PV}", pv, needed_file)
+                    self.print_cross_files(bbpath, keyword, layername, f, needed_file, show_filenames)
+                line = fnfile.readline()
+            fnfile.close()
+
+        # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
+        conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
+        inc_re = re.compile(".*\.inc$")
+        # The "inherit xxx" in .bbclass
+        bbclass_re = re.compile(".*\.bbclass$")
+        for layerdir in self.bblayers:
+            layername = self.get_layer_name(layerdir)
+            for dirpath, dirnames, filenames in os.walk(layerdir):
+                for name in filenames:
+                    f = os.path.join(dirpath, name)
+                    s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
+                    if s:
+                        ffile = open(f, 'r')
+                        line = ffile.readline()
+                        while line:
+                            m, keyword = self.match_require_include(line)
+                            # Only bbclass has the "inherit xxx" here.
+                            bbclass=""
+                            if not m and f.endswith(".bbclass"):
+                                m, keyword = self.match_inherit(line)
+                                bbclass=".bbclass"
+                            # Find a 'require/include xxxx'
+                            if m:
+                                self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, show_filenames)
+                            line = ffile.readline()
+                        ffile.close()
+
+    def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames):
+        """Print the depends that crosses a layer boundary"""
+        needed_file = bb.utils.which(bbpath, needed_filename)
+        if needed_file:
+            # Which layer is this file from
+            needed_layername = self.get_file_layer(needed_file)
+            if needed_layername != layername:
+                if not show_filenames:
+                    f = self.remove_layer_prefix(f)
+                    needed_file = self.remove_layer_prefix(needed_file)
+                logger.plain("%s %s %s" %(f, keyword, needed_file))
+    def match_inherit(self, line):
+        """Match the inherit xxx line"""
+        return (self.inherit_re.match(line), "inherits")
+
+    def match_require_include(self, line):
+        """Match the require/include xxx line"""
+        m = self.require_re.match(line)
+        keyword = "requires"
+        if not m:
+            m = self.include_re.match(line)
+            keyword = "includes"
+        return (m, keyword)
+
+    def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames):
+        """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
+        best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
+        needed_layername = self.get_file_layer(best_realfn)
+        if needed_layername != layername:
+            if not show_filenames:
+                f = self.remove_layer_prefix(f)
+                best_realfn = self.remove_layer_prefix(best_realfn)
+
+            logger.plain("%s %s %s" % (f, keyword, best_realfn))
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]) or 0)
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv
new file mode 100755
index 0000000000..a8d7acb4c2
--- /dev/null
+++ b/bitbake/bin/bitbake-prserv
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+import os
+import sys,logging
+import optparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
+
+import prserv
+import prserv.serv
+
+__version__="1.0.0"
+
+PRHOST_DEFAULT='0.0.0.0'
+PRPORT_DEFAULT=8585
+
+def main():
+    parser = optparse.OptionParser(
+        version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
+        usage = "%prog < --start | --stop > [options]")
+
+    parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store",
+                      dest="dbfile", type="string", default="prserv.sqlite3")
+    parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store",
+                      dest="logfile", type="string", default="prserv.log")
+    parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
+                      action = "store", type="string", dest="loglevel", default = "INFO")
+    parser.add_option("--start", help="start daemon",
+                      action="store_true", dest="start")
+    parser.add_option("--stop", help="stop daemon",
+                      action="store_true", dest="stop")
+    parser.add_option("--host", help="ip address to bind", action="store",
+                      dest="host", type="string", default=PRHOST_DEFAULT)
+    parser.add_option("--port", help="port number(default: 8585)", action="store",
+                      dest="port", type="int", default=PRPORT_DEFAULT)
+
+    options, args = parser.parse_args(sys.argv)
+    prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
+
+    if options.start:
+        ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile))
+    elif options.stop:
+        ret=prserv.serv.stop_daemon(options.host, options.port)
+    else:
+        ret=parser.print_help()
+    return ret
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc(5)
+    sys.exit(ret)
+
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
new file mode 100755
index 0000000000..48a58fef67
--- /dev/null
+++ b/bitbake/bin/bitbake-selftest
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys, logging
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+
+import unittest
+try:
+    import bb
+except RuntimeError as exc:
+    sys.exit(str(exc))
+
+tests = ["bb.tests.codeparser", 
+         "bb.tests.cow",
+         "bb.tests.data",
+         "bb.tests.fetch",
+         "bb.tests.utils"]
+
+for t in tests:
+    __import__(t)
+
+unittest.main(argv=["bitbake-selftest"] + tests)
+
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
new file mode 100755
index 0000000000..66b6aabfdb
--- /dev/null
+++ b/bitbake/bin/bitbake-worker
@@ -0,0 +1,363 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import warnings
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+from bb import fetch2
+import logging
+import bb
+import select
+import errno
+import signal
+
+# Users shouldn't be running this code directly
+if len(sys.argv) != 2 or sys.argv[1] != "decafbad":
+    print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
+    sys.exit(1)
+
+logger = logging.getLogger("BitBake")
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+    bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
+
+
+worker_pipe = sys.stdout.fileno()
+bb.utils.nonblockingfd(worker_pipe)
+
+handler = bb.event.LogHandler()
+logger.addHandler(handler)
+
+if 0:
+    # Code to write out a log file of all events passing through the worker
+    logfilename = "/tmp/workerlogfile"
+    format_str = "%(levelname)s: %(message)s"
+    conlogformat = bb.msg.BBLogFormatter(format_str)
+    consolelog = logging.FileHandler(logfilename)
+    bb.msg.addDefaultlogFilter(consolelog)
+    consolelog.setFormatter(conlogformat)
+    logger.addHandler(consolelog)
+
+worker_queue = ""
+
+def worker_fire(event, d):
+    data = "<event>" + pickle.dumps(event) + "</event>"
+    worker_fire_prepickled(data)
+
+def worker_fire_prepickled(event):
+    global worker_queue
+
+    worker_queue = worker_queue + event
+    worker_flush()
+
+def worker_flush():
+    global worker_queue, worker_pipe
+
+    if not worker_queue:
+        return
+
+    try:
+        written = os.write(worker_pipe, worker_queue)
+        worker_queue = worker_queue[written:]
+    except (IOError, OSError) as e:
+        if e.errno != errno.EAGAIN:
+            raise
+
+def worker_child_fire(event, d):
+    global worker_pipe
+
+    data = "<event>" + pickle.dumps(event) + "</event>"
+    worker_pipe.write(data)
+
+bb.event.worker_fire = worker_fire
+
+lf = None
+#lf = open("/tmp/workercommandlog", "w+")
+def workerlog_write(msg):
+    if lf:
+        lf.write(msg)
+        lf.flush()
+
+def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterrors=False):
+    # We need to setup the environment BEFORE the fork, since
+    # a fork() or exec*() activates PSEUDO...
+
+    envbackup = {}
+    fakeenv = {}
+    umask = None
+
+    taskdep = workerdata["taskdeps"][fn]
+    if 'umask' in taskdep and taskname in taskdep['umask']:
+        # umask might come in as a number or text string..
+        try:
+             umask = int(taskdep['umask'][taskname],8)
+        except TypeError:
+             umask = taskdep['umask'][taskname]
+
+    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
+        envvars = (workerdata["fakerootenv"][fn] or "").split()
+        for key, value in (var.split('=') for var in envvars):
+            envbackup[key] = os.environ.get(key)
+            os.environ[key] = value
+            fakeenv[key] = value
+
+        fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
+        for p in fakedirs:
+            bb.utils.mkdirhier(p)
+        logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' %
+                        (fn, taskname, ', '.join(fakedirs)))
+    else:
+        envvars = (workerdata["fakerootnoenv"][fn] or "").split()
+        for key, value in (var.split('=') for var in envvars):
+            envbackup[key] = os.environ.get(key)
+            os.environ[key] = value
+            fakeenv[key] = value
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    try:
+        pipein, pipeout = os.pipe()
+        pipein = os.fdopen(pipein, 'rb', 4096)
+        pipeout = os.fdopen(pipeout, 'wb', 0)
+        pid = os.fork()
+    except OSError as e:
+        bb.msg.fatal("RunQueue", "fork failed: %d (%s)" % (e.errno, e.strerror))
+
+    if pid == 0:
+            global worker_pipe
+            pipein.close()
+
+            # Save out the PID so that the event can include it the
+            # events
+            bb.event.worker_pid = os.getpid()
+            bb.event.worker_fire = worker_child_fire
+            worker_pipe = pipeout
+
+            # Make the child the process group leader
+            os.setpgid(0, 0)
+            # No stdin
+            newsi = os.open(os.devnull, os.O_RDWR)
+            os.dup2(newsi, sys.stdin.fileno())
+
+            if umask:
+                os.umask(umask)
+
+            data.setVar("BB_WORKERCONTEXT", "1")
+            data.setVar("BUILDNAME", workerdata["buildname"])
+            data.setVar("DATE", workerdata["date"])
+            data.setVar("TIME", workerdata["time"])
+            bb.parse.siggen.set_taskdata(workerdata["hashes"], workerdata["hash_deps"], workerdata["sigchecksums"])
+            ret = 0
+            try:
+                the_data = bb.cache.Cache.loadDataFull(fn, appends, data)
+                the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
+                for h in workerdata["hashes"]:
+                    the_data.setVar("BBHASH_%s" % h, workerdata["hashes"][h])
+                for h in workerdata["hash_deps"]:
+                    the_data.setVar("BBHASHDEPS_%s" % h, workerdata["hash_deps"][h])
+
+                # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
+                # successfully. We also need to unset anything from the environment which shouldn't be there 
+                exports = bb.data.exported_vars(the_data)
+                bb.utils.empty_environment()
+                for e, v in exports:
+                    os.environ[e] = v
+                for e in fakeenv:
+                    os.environ[e] = fakeenv[e]
+                    the_data.setVar(e, fakeenv[e])
+                    the_data.setVarFlag(e, 'export', "1")
+
+                if quieterrors:
+                    the_data.setVarFlag(taskname, "quieterrors", "1")
+
+            except Exception as exc:
+                if not quieterrors:
+                    logger.critical(str(exc))
+                os._exit(1)
+            try:
+                if not cfg.dry_run:
+                    ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                os._exit(ret)
+            except:
+                os._exit(1)
+    else:
+        for key, value in envbackup.iteritems():
+            if value is None:
+                del os.environ[key]
+            else:
+                os.environ[key] = value
+
+    return pid, pipein, pipeout
+
+class runQueueWorkerPipe():
+    """
+    Abstraction for a pipe between a worker thread and the worker server
+    """
+    def __init__(self, pipein, pipeout):
+        self.input = pipein
+        if pipeout:
+            pipeout.close()
+        bb.utils.nonblockingfd(self.input)
+        self.queue = ""
+
+    def read(self):
+        start = len(self.queue)
+        try:
+            self.queue = self.queue + self.input.read(102400)
+        except (OSError, IOError) as e:
+            if e.errno != errno.EAGAIN:
+                raise
+
+        end = len(self.queue)
+        index = self.queue.find("</event>")
+        while index != -1:
+            worker_fire_prepickled(self.queue[:index+8])
+            self.queue = self.queue[index+8:]
+            index = self.queue.find("</event>")
+        return (end > start)
+
+    def close(self):
+        while self.read():
+            continue
+        if len(self.queue) > 0:
+            print("Warning, worker child left partial message: %s" % self.queue)
+        self.input.close()
+
+normalexit = False
+
+class BitbakeWorker(object):
+    def __init__(self, din):
+        self.input = din
+        bb.utils.nonblockingfd(self.input)
+        self.queue = ""
+        self.cookercfg = None
+        self.databuilder = None
+        self.data = None
+        self.build_pids = {}
+        self.build_pipes = {}
+    
+    def serve(self):        
+        while True:
+            (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
+            if self.input in ready or len(self.queue):
+                start = len(self.queue)
+                try:
+                    self.queue = self.queue + self.input.read()
+                except (OSError, IOError):
+                    pass
+                end = len(self.queue)
+                self.handle_item("cookerconfig", self.handle_cookercfg)
+                self.handle_item("workerdata", self.handle_workerdata)
+                self.handle_item("runtask", self.handle_runtask)
+                self.handle_item("finishnow", self.handle_finishnow)
+                self.handle_item("ping", self.handle_ping)
+                self.handle_item("quit", self.handle_quit)
+
+            for pipe in self.build_pipes:
+                self.build_pipes[pipe].read()
+            if len(self.build_pids):
+                self.process_waitpid()
+            worker_flush()
+
+
+    def handle_item(self, item, func):
+        if self.queue.startswith("<" + item + ">"):
+            index = self.queue.find("</" + item + ">")
+            while index != -1:
+                func(self.queue[(len(item) + 2):index])
+                self.queue = self.queue[(index + len(item) + 3):]
+                index = self.queue.find("</" + item + ">")
+
+    def handle_cookercfg(self, data):
+        self.cookercfg = pickle.loads(data)
+        self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
+        self.databuilder.parseBaseConfiguration()
+        self.data = self.databuilder.data
+
+    def handle_workerdata(self, data):
+        self.workerdata = pickle.loads(data)
+        bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
+        bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
+        bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
+        bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
+        self.data.setVar("PRSERV_HOST", self.workerdata["prhost"])
+
+    def handle_ping(self, _):
+        workerlog_write("Handling ping\n")
+
+        logger.warn("Pong from bitbake-worker!")
+
+    def handle_quit(self, data):
+        workerlog_write("Handling quit\n")
+
+        global normalexit
+        normalexit = True
+        sys.exit(0)
+
+    def handle_runtask(self, data):
+        fn, task, taskname, quieterrors, appends = pickle.loads(data)
+        workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
+
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, quieterrors)
+
+        self.build_pids[pid] = task
+        self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
+
+    def process_waitpid(self):
+        """
+        Return none is there are no processes awaiting result collection, otherwise
+        collect the process exit codes and close the information pipe.
+        """
+        try:
+            pid, status = os.waitpid(-1, os.WNOHANG)
+            if pid == 0 or os.WIFSTOPPED(status):
+                return None
+        except OSError:
+            return None
+
+        workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
+
+        if os.WIFEXITED(status):
+            status = os.WEXITSTATUS(status)
+        elif os.WIFSIGNALED(status):
+            # Per shell conventions for $?, when a process exits due to
+            # a signal, we return an exit code of 128 + SIGNUM
+            status = 128 + os.WTERMSIG(status)
+
+        task = self.build_pids[pid]
+        del self.build_pids[pid]
+
+        self.build_pipes[pid].close()
+        del self.build_pipes[pid]
+
+        worker_fire_prepickled("<exitcode>" + pickle.dumps((task, status)) + "</exitcode>")
+
+    def handle_finishnow(self, _):
+        if self.build_pids:
+            logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
+            for k, v in self.build_pids.iteritems():
+                try:
+                    os.kill(-k, signal.SIGTERM)
+                    os.waitpid(-1, 0)
+                except:
+                    pass
+        for pipe in self.build_pipes:
+            self.build_pipes[pipe].read()
+
+try:
+    worker = BitbakeWorker(sys.stdin)
+    worker.serve()
+except BaseException as e:
+    if not normalexit:
+        import traceback
+        sys.stderr.write(traceback.format_exc())
+        sys.stderr.write(str(e))
+while len(worker_queue):
+    worker_flush()
+workerlog_write("exitting")
+sys.exit(0)
+
diff --git a/bitbake/bin/bitdoc b/bitbake/bin/bitdoc
new file mode 100755
index 0000000000..576d88b574
--- /dev/null
+++ b/bitbake/bin/bitdoc
@@ -0,0 +1,531 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2005 Holger Hans Peter Freyther
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import optparse, os, sys
+
+# bitbake
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__), 'lib'))
+import bb
+import bb.parse
+from   string import split, join
+
+__version__ = "0.0.2"
+
+class HTMLFormatter:
+    """
+    Simple class to help to generate some sort of HTML files. It is
+    quite inferior solution compared to docbook, gtkdoc, doxygen but it
+    should work for now.
+    We've a global introduction site (index.html) and then one site for
+    the list of keys (alphabetical sorted) and one for the list of groups,
+    one site for each key with links to the relations and groups.
+
+        index.html
+        all_keys.html
+        all_groups.html
+        groupNAME.html
+        keyNAME.html
+    """
+
+    def replace(self, text, *pairs):
+        """
+        From pydoc... almost identical at least
+        """
+        while pairs:
+            (a, b) = pairs[0]
+            text = join(split(text, a), b)
+            pairs = pairs[1:]
+        return text
+    def escape(self, text):
+        """
+        Escape string to be conform HTML
+        """
+        return self.replace(text, 
+                            ('&', '&amp;'), 
+                            ('<', '&lt;' ),
+                            ('>', '&gt;' ) )
+    def createNavigator(self):
+        """
+        Create the navgiator
+        """
+        return """<table class="navigation" width="100%" summary="Navigation header" cellpadding="2" cellspacing="2">
+<tr valign="middle">
+<td><a accesskey="g" href="index.html">Home</a></td>
+<td><a accesskey="n" href="all_groups.html">Groups</a></td>
+<td><a accesskey="u" href="all_keys.html">Keys</a></td>
+</tr></table>
+"""
+
+    def relatedKeys(self, item):
+        """
+        Create HTML to link to foreign keys
+        """
+
+        if len(item.related()) == 0:
+            return ""
+
+        txt = "<p><b>See also:</b><br>"
+        txts = []
+        for it in item.related():
+            txts.append("""<a href="key%(it)s.html">%(it)s</a>""" % vars() )
+
+        return txt + ",".join(txts)
+
+    def groups(self, item):
+        """
+        Create HTML to link to related groups
+        """
+
+        if len(item.groups()) == 0:
+            return ""
+
+
+        txt = "<p><b>See also:</b><br>"
+        txts = []
+        for group in item.groups():
+            txts.append( """<a href="group%s.html">%s</a> """ % (group, group) )
+
+        return txt + ",".join(txts)
+
+
+    def createKeySite(self, item):
+        """
+        Create a site for a key. It contains the header/navigator, a heading,
+        the description, links to related keys and to the groups.
+        """
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Key %s</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2><span class="refentrytitle">%s</span></h2>
+
+<div class="refsynopsisdiv">
+<h2>Synopsis</h2>
+<p>
+%s
+</p>
+</div>
+
+<div class="refsynopsisdiv">
+<h2>Related Keys</h2>
+<p>
+%s
+</p>
+</div>
+
+<div class="refsynopsisdiv">
+<h2>Groups</h2>
+<p>
+%s
+</p>
+</div>
+
+
+</body>
+"""     % (item.name(), self.createNavigator(), item.name(), 
+           self.escape(item.description()), self.relatedKeys(item), self.groups(item))
+
+    def createGroupsSite(self, doc):
+        """
+        Create the Group Overview site
+        """
+
+        groups = ""
+        sorted_groups = sorted(doc.groups())
+        for group in sorted_groups:
+            groups += """<a href="group%s.html">%s</a><br>""" % (group, group)
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Group overview</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Available Groups</h2>
+%s
+</body>
+""" % (self.createNavigator(), groups)
+
+    def createIndex(self):
+        """
+        Create the index file
+        """
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Bitbake Documentation</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Documentation Entrance</h2>
+<a href="all_groups.html">All available groups</a><br>
+<a href="all_keys.html">All available keys</a><br>
+</body>
+""" % self.createNavigator()
+
+    def createKeysSite(self, doc):
+        """
+        Create Overview of all avilable keys
+        """
+        keys = ""
+        sorted_keys = sorted(doc.doc_keys())
+        for key in sorted_keys:
+            keys += """<a href="key%s.html">%s</a><br>""" % (key, key)
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Key overview</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Available Keys</h2>
+%s
+</body>
+""" % (self.createNavigator(), keys)
+
+    def createGroupSite(self, gr, items, _description = None):
+        """
+        Create a site for a group:
+        Group the name of the group, items contain the name of the keys
+        inside this group
+        """
+        groups = ""
+        description = ""
+
+        # create a section with the group descriptions
+        if _description:
+            description  += "<h2 Description of Grozp %s</h2>" % gr
+            description  += _description
+
+        items.sort(lambda x, y:cmp(x.name(), y.name()))
+        for group in items:
+            groups += """<a href="key%s.html">%s</a><br>""" % (group.name(), group.name())
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Group %s</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+%s
+<div class="refsynopsisdiv">
+<h2>Keys in Group %s</h2>
+<pre class="synopsis">
+%s
+</pre>
+</div>
+</body>
+""" % (gr, self.createNavigator(), description, gr, groups)
+
+
+
+    def createCSS(self):
+        """
+        Create the CSS file
+        """
+        return """.synopsis, .classsynopsis
+{
+  background: #eeeeee;
+  border: solid 1px #aaaaaa;
+  padding: 0.5em;
+}
+.programlisting
+{
+  background: #eeeeff;
+  border: solid 1px #aaaaff;
+  padding: 0.5em;
+}
+.variablelist
+{
+  padding: 4px;
+  margin-left: 3em;
+}
+.variablelist td:first-child
+{
+  vertical-align: top;
+}
+table.navigation
+{
+  background: #ffeeee;
+  border: solid 1px #ffaaaa;
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+.navigation a
+{
+  color: #770000;
+}
+.navigation a:visited
+{
+  color: #550000;
+}
+.navigation .title
+{
+  font-size: 200%;
+}
+div.refnamediv
+{
+  margin-top: 2em;
+}
+div.gallery-float
+{
+  float: left;
+  padding: 10px;
+}
+div.gallery-float img
+{
+  border-style: none;
+}
+div.gallery-spacer
+{
+  clear: both;
+}
+a
+{
+  text-decoration: none;
+}
+a:hover
+{
+  text-decoration: underline;
+  color: #FF0000;
+}
+"""
+
+
+
+class DocumentationItem:
+    """
+    A class to hold information about a configuration
+    item. It contains the key name, description, a list of related names,
+    and the group this item is contained in.
+    """
+
+    def __init__(self):
+        self._groups  = []
+        self._related = []
+        self._name    = ""
+        self._desc    = ""
+
+    def groups(self):
+        return self._groups
+
+    def name(self):
+        return self._name
+
+    def description(self):
+        return self._desc
+
+    def related(self):
+        return self._related
+
+    def setName(self, name):
+        self._name = name
+
+    def setDescription(self, desc):
+        self._desc = desc
+
+    def addGroup(self, group):
+        self._groups.append(group)
+
+    def addRelation(self, relation):
+        self._related.append(relation)
+
+    def sort(self):
+        self._related.sort()
+        self._groups.sort()
+
+
+class Documentation:
+    """
+    Holds the documentation... with mappings from key to items...
+    """
+
+    def __init__(self):
+        self.__keys   = {}
+        self.__groups = {}
+
+    def insert_doc_item(self, item):
+        """
+        Insert the Doc Item into the internal list
+        of representation
+        """
+        item.sort()
+        self.__keys[item.name()] = item
+
+        for group in item.groups():
+            if not group in self.__groups:
+                self.__groups[group] = []
+            self.__groups[group].append(item)
+            self.__groups[group].sort()
+
+
+    def doc_item(self, key):
+        """
+        Return the DocumentationInstance describing the key
+        """
+        try:
+            return self.__keys[key]
+        except KeyError:
+            return None
+
+    def doc_keys(self):
+        """
+        Return the documented KEYS (names)
+        """
+        return self.__keys.keys()
+
+    def groups(self):
+        """
+        Return the names of available groups
+        """
+        return self.__groups.keys()
+
+    def group_content(self, group_name):
+        """
+        Return a list of keys/names that are in a specefic
+        group or the empty list
+        """
+        try:
+            return self.__groups[group_name]
+        except KeyError:
+            return []
+
+
+def parse_cmdline(args):
+    """
+    Parse the CMD line and return the result as a n-tuple
+    """
+
+    parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__, __version__))
+    usage  = """%prog [options]
+
+Create a set of html pages (documentation) for a bitbake.conf....
+"""
+
+    # Add the needed options
+    parser.add_option( "-c", "--config", help = "Use the specified configuration file as source",
+                       action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") )
+
+    parser.add_option( "-o", "--output", help = "Output directory for html files",
+                       action = "store", dest = "output", default = "html/" )
+
+    parser.add_option( "-D",  "--debug", help = "Increase the debug level",
+                       action = "count", dest = "debug", default = 0 )
+
+    parser.add_option( "-v", "--verbose", help = "output more chit-char to the terminal",
+                       action = "store_true", dest = "verbose", default = False )
+
+    options, args = parser.parse_args( sys.argv )
+ 
+    bb.msg.init_msgconfig(options.verbose, options.debug)
+
+    return options.config, options.output
+
+def main():
+    """
+    The main Method
+    """
+
+    (config_file, output_dir) = parse_cmdline( sys.argv )
+
+    # right to let us load the file now
+    try:
+        documentation = bb.parse.handle( config_file, bb.data.init() )
+    except IOError:
+        bb.fatal( "Unable to open %s" % config_file )
+    except bb.parse.ParseError:
+        bb.fatal( "Unable to parse %s" % config_file )
+
+    if isinstance(documentation, dict):
+        documentation = documentation[""]
+
+    # Assuming we've the file loaded now, we will initialize the 'tree'
+    doc = Documentation()
+
+    # defined states
+    state_begin = 0
+    state_see   = 1
+    state_group = 2
+
+    for key in bb.data.keys(documentation):
+        data   = documentation.getVarFlag(key, "doc")
+        if not data:
+            continue
+
+        # The Documentation now starts
+        doc_ins = DocumentationItem()
+        doc_ins.setName(key)
+
+
+        tokens = data.split(' ')
+        state = state_begin
+        string= ""
+        for token in tokens:
+            token = token.strip(',')
+
+            if not state == state_see and token == "@see":
+                state = state_see
+                continue
+            elif not state == state_group and token  == "@group":
+                state = state_group
+                continue
+
+            if state == state_begin:
+                string += " %s" % token
+            elif state == state_see:
+                doc_ins.addRelation(token)
+            elif state == state_group:
+                doc_ins.addGroup(token)
+
+        # set the description
+        doc_ins.setDescription(string)
+        doc.insert_doc_item(doc_ins)
+
+    # let us create the HTML now
+    bb.utils.mkdirhier(output_dir)
+    os.chdir(output_dir)
+
+    # Let us create the sites now. We do it in the following order
+    # Start with the index.html. It will point to sites explaining all
+    # keys and groups
+    html_slave = HTMLFormatter()
+
+    f = file('style.css', 'w')
+    print >> f, html_slave.createCSS()
+
+    f = file('index.html', 'w')
+    print >> f, html_slave.createIndex()
+
+    f = file('all_groups.html', 'w')
+    print >> f, html_slave.createGroupsSite(doc)
+
+    f = file('all_keys.html', 'w')
+    print >> f, html_slave.createKeysSite(doc)
+
+    # now for each group create the site
+    for group in doc.groups():
+        f = file('group%s.html' % group, 'w')
+        print >> f, html_slave.createGroupSite(group, doc.group_content(group))
+
+    # now for the keys
+    for key in doc.doc_keys():
+        f = file('key%s.html' % doc.doc_item(key).name(), 'w')
+        print >> f, html_slave.createKeySite(doc.doc_item(key))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bitbake/bin/image-writer b/bitbake/bin/image-writer
new file mode 100755
index 0000000000..86c38b5769
--- /dev/null
+++ b/bitbake/bin/image-writer
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2012 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname( \
+                                    os.path.abspath(__file__))), 'lib'))
+try:
+    import bb
+except RuntimeError as exc:
+    sys.exit(str(exc))
+
+import gtk
+import optparse
+import pygtk
+
+from bb.ui.crumbs.hobwidget import HobAltButton, HobButton
+from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog
+from bb.ui.crumbs.hig.deployimagedialog import DeployImageDialog
+from bb.ui.crumbs.hig.imageselectiondialog import ImageSelectionDialog
+
+# I put all the fs bitbake supported here. Need more test.
+DEPLOYABLE_IMAGE_TYPES = ["jffs2", "cramfs", "ext2", "ext3", "btrfs", "squashfs", "ubi", "vmdk"]
+Title = "USB Image Writer"
+
+class DeployWindow(gtk.Window):
+    def __init__(self, image_path=''):
+        super(DeployWindow, self).__init__()
+
+        if len(image_path) > 0:
+            valid = True
+            if not os.path.exists(image_path):
+                valid = False
+                lbl = "<b>Invalid image file path: %s.</b>\nPress <b>Select Image</b> to select an image." % image_path
+            else:
+                image_path = os.path.abspath(image_path)
+                extend_name = os.path.splitext(image_path)[1][1:]
+                if extend_name not in DEPLOYABLE_IMAGE_TYPES:
+                    valid = False
+                    lbl = "<b>Undeployable imge type: %s</b>\nPress <b>Select Image</b> to select an image." % extend_name
+
+            if not valid:
+                image_path = ''
+                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+                button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK)
+                HobButton.style_button(button)
+                crumbs_dialog.run()
+                crumbs_dialog.destroy()
+
+        self.deploy_dialog = DeployImageDialog(Title, image_path, self,
+                                        gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT
+                                        | gtk.DIALOG_NO_SEPARATOR, None, standalone=True)
+        close_button = self.deploy_dialog.add_button("Close", gtk.RESPONSE_NO)
+        HobAltButton.style_button(close_button)
+        close_button.connect('clicked', gtk.main_quit)
+
+        write_button = self.deploy_dialog.add_button("Write USB image", gtk.RESPONSE_YES)
+        HobAltButton.style_button(write_button)
+
+        self.deploy_dialog.connect('select_image_clicked', self.select_image_clicked_cb)
+        self.deploy_dialog.connect('destroy', gtk.main_quit)
+        response = self.deploy_dialog.show()
+
+    def select_image_clicked_cb(self, dialog):
+        cwd = os.getcwd()
+        dialog = ImageSelectionDialog(cwd, DEPLOYABLE_IMAGE_TYPES, Title, self, gtk.FILE_CHOOSER_ACTION_SAVE )
+        button = dialog.add_button("Cancel", gtk.RESPONSE_NO)
+        HobAltButton.style_button(button)
+        button = dialog.add_button("Open", gtk.RESPONSE_YES)
+        HobAltButton.style_button(button)
+        response = dialog.run()
+
+        if response == gtk.RESPONSE_YES:
+            if not dialog.image_names:
+                lbl = "<b>No selections made</b>\nClicked the radio button to select a image."
+                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+                button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK)
+                HobButton.style_button(button)
+                crumbs_dialog.run()
+                crumbs_dialog.destroy()
+                dialog.destroy()
+                return
+
+            # get the full path of image
+            image_path = os.path.join(dialog.image_folder, dialog.image_names[0])
+            self.deploy_dialog.set_image_text_buffer(image_path)
+            self.deploy_dialog.set_image_path(image_path)
+
+        dialog.destroy()
+
+def main():
+    parser = optparse.OptionParser(
+                usage = """%prog [-h] [image_file]
+
+%prog writes bootable images to USB devices. You can
+provide the image file on the command line or select it using the GUI.""")
+
+    options, args = parser.parse_args(sys.argv)
+    image_file = args[1] if len(args) > 1 else ''
+    dw = DeployWindow(image_file)
+
+if __name__ == '__main__':
+    try:
+        main()
+        gtk.main()
+    except Exception:
+        import traceback
+        traceback.print_exc(3)