47 files changed, 1599 insertions, 1788 deletions
diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py
index ca206cf4b4..6917ec378a 100644
--- a/bitbake/lib/bb/COW.py
+++ b/bitbake/lib/bb/COW.py
@@ -3,7 +3,7 @@
 #
 # This is a copy on write dictionary and set which abuses classes to try and be nice and fast.
 #
-# Copyright (C) 2006 Tim Amsell 
+# Copyright (C) 2006 Tim Amsell
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -18,29 +18,31 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-#Please Note: 
+#Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.
 #
+from __future__ import print_function
 import copy
 import types
-types.ImmutableTypes = tuple([ \
+ImmutableTypes = (
-    types.BooleanType, \
+    types.NoneType,
-    types.ComplexType, \
+    bool,
-    types.FloatType, \
+    complex,
-    types.IntType, \
+    float,
-    types.LongType, \
+    int,
-    types.NoneType, \
+    long,
-    types.TupleType, \
+    tuple,
-    frozenset] + \
+    frozenset,
-    list(types.StringTypes))
+    basestring
+)
 MUTABLE = "__mutable__"
 class COWMeta(type):
    pass
- 
 class COWDictMeta(COWMeta):
    __warn__ = False
    __hasmutable__ = False
@@ -59,12 +61,12 @@ class COWDictMeta(COWMeta):
    __call__ = cow
    def __setitem__(cls, key, value):
-        if not isinstance(value, types.ImmutableTypes):
+        if not isinstance(value, ImmutableTypes):
            if not isinstance(value, COWMeta):
                cls.__hasmutable__ = True
            key += MUTABLE
        setattr(cls, key, value)
-    
    def __getmutable__(cls, key, readonly=False):
        nkey = key + MUTABLE
        try:
@@ -77,10 +79,10 @@ class COWDictMeta(COWMeta):
            return value
        if not cls.__warn__ is False and not isinstance(value, COWMeta):
-            print >> cls.__warn__, "Warning: Doing a copy because %s is a mutable type." % key
+            print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__)
        try:
            value = value.copy()
-        except AttributeError, e:
+        except AttributeError as e:
            value = copy.copy(value)
        setattr(cls, nkey, value)
        return value
@@ -98,13 +100,13 @@ class COWDictMeta(COWMeta):
                value = getattr(cls, key)
            except AttributeError:
                value = cls.__getmutable__(key, readonly)
-            
-            # This is for values which have been deleted 
+            # This is for values which have been deleted
            if value is cls.__marker__:
                raise AttributeError("key %s does not exist." % key)
            return value
-        except AttributeError, e:
+        except AttributeError as e:
            if not default is cls.__getmarker__:
                return default
@@ -118,6 +120,9 @@ class COWDictMeta(COWMeta):
            key += MUTABLE
        delattr(cls, key)
+    def __contains__(cls, key):
+        return cls.has_key(key)
    def has_key(cls, key):
        value = cls.__getreadonly__(key, cls.__marker__)
        if value is cls.__marker__:
@@ -127,7 +132,7 @@ class COWDictMeta(COWMeta):
    def iter(cls, type, readonly=False):
        for key in dir(cls):
            if key.startswith("__"):
-                    continue
+                continue
            if key.endswith(MUTABLE):
                key = key[:-len(MUTABLE)]
@@ -153,11 +158,11 @@ class COWDictMeta(COWMeta):
        return cls.iter("keys")
    def itervalues(cls, readonly=False):
        if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
-            print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
+            print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__)
        return cls.iter("values", readonly)
    def iteritems(cls, readonly=False):
        if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
-            print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
+            print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__)
        return cls.iter("items", readonly)
 class COWSetMeta(COWDictMeta):
@@ -176,13 +181,13 @@ class COWSetMeta(COWDictMeta):
    def remove(cls, value):
        COWDictMeta.__delitem__(cls, repr(hash(value)))
- 
    def __in__(cls, value):
        return COWDictMeta.has_key(repr(hash(value)))
    def iterkeys(cls):
        raise TypeError("sets don't have keys")
-    
    def iteritems(cls):
        raise TypeError("sets don't have 'items'")
@@ -199,120 +204,120 @@ if __name__ == "__main__":
    import sys
    COWDictBase.__warn__ = sys.stderr
    a = COWDictBase()
-    print "a", a
+    print("a", a)
    a['a'] = 'a'
    a['b'] = 'b'
    a['dict'] = {}
    b = a.copy()
-    print "b", b
+    print("b", b)
    b['c'] = 'b'
-    print
+    print()
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems():
-        print x
+        print(x)
-    print
+    print()
    b['dict']['a'] = 'b'
    b['a'] = 'c'
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems():
-        print x
+        print(x)
-    print
+    print()
    try:
        b['dict2']
-    except KeyError, e:
+    except KeyError as e:
-        print "Okay!"
+        print("Okay!")
    a['set'] = COWSetBase()
    a['set'].add("o1")
    a['set'].add("o1")
    a['set'].add("o2")
-    print "a", a
+    print("a", a)
    for x in a['set'].itervalues():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b['set'].itervalues():
-        print x
+        print(x)
-    print
+    print()
    b['set'].add('o3')
-    print "a", a
+    print("a", a)
    for x in a['set'].itervalues():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b['set'].itervalues():
-        print x
+        print(x)
-    print
+    print()
    a['set2'] = set()
    a['set2'].add("o1")
    a['set2'].add("o1")
    a['set2'].add("o2")
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems(readonly=True):
-        print x
+        print(x)
-    print
+    print()
    del b['b']
    try:
-        print b['b']
+        print(b['b'])
    except KeyError:
-        print "Yay! deleted key raises error"
+        print("Yay! deleted key raises error")
    if b.has_key('b'):
-        print "Boo!"
+        print("Boo!")
    else:
-        print "Yay - has_key with delete works!"
+        print("Yay - has_key with delete works!")
-    
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems(readonly=True):
-        print x
+        print(x)
-    print
+    print()
    b.__revertitem__('b')
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems(readonly=True):
-        print x
+        print(x)
-    print
+    print()
    b.__revertitem__('dict')
-    print "a", a
+    print("a", a)
    for x in a.iteritems():
-        print x
+        print(x)
-    print "--"
+    print("--")
-    print "b", b
+    print("b", b)
    for x in b.iteritems(readonly=True):
-        print x
+        print(x)
-    print
+    print()
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index c7cd0f62dd..88adfc1dfa 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -21,39 +21,14 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-__version__ = "1.9.0"
+__version__ = "1.11.0"
-__all__ = [
+import sys
+if sys.version_info < (2, 6, 0):
-    "debug",
+    raise RuntimeError("Sorry, python 2.6.0 or later is required for this version of bitbake")
-    "note",
-    "error",
+import os
-    "fatal",
+import bb.msg
-    "mkdirhier",
-    "movefile",
-    "vercmp",
-# fetch
-    "decodeurl",
-    "encodeurl",
-# modules
-    "parse",
-    "data",
-    "command",
-    "event",
-    "build",
-    "fetch",
-    "manifest",
-    "methodpool",
-    "cache",
-    "runqueue",
-    "taskdata",
-    "providers",
- ]
-import sys, os, types, re, string
 if "BBDEBUG" in os.environ:
    level = int(os.environ["BBDEBUG"])
@@ -81,14 +56,45 @@ def fatal(*args):
    bb.msg.fatal(None, ''.join(args))
-# For compatibility
+def deprecated(func, name = None, advice = ""):
-from bb.fetch import MalformedUrl, encodeurl, decodeurl
+    """This is a decorator which can be used to mark functions
-from bb.data import VarExpandError
+    as deprecated. It will result in a warning being emmitted
-from bb.utils import mkdirhier, movefile, copyfile, which
+    when the function is used."""
-from bb.utils import vercmp
+    import warnings
+    if advice:
+        advice = ": %s" % advice
+    if name is None:
+        name = func.__name__
+    def newFunc(*args, **kwargs):
+        warnings.warn("Call to deprecated function %s%s." % (name,
+                                                             advice),
+                      category = PendingDeprecationWarning,
+                      stacklevel = 2)
+        return func(*args, **kwargs)
+    newFunc.__name__ = func.__name__
+    newFunc.__doc__ = func.__doc__
+    newFunc.__dict__.update(func.__dict__)
+    return newFunc
+# For compatibility
-if __name__ == "__main__":
+def deprecate_import(current, modulename, fromlist, renames = None):
-    import doctest, bb
+    """Import objects from one module into another, wrapping them with a DeprecationWarning"""
-    bb.msg.set_debug_level(0)
+    import sys
-    doctest.testmod(bb)
+    module = __import__(modulename, fromlist = fromlist)
+    for position, objname in enumerate(fromlist):
+        obj = getattr(module, objname)
+        newobj = deprecated(obj, "{0}.{1}".format(current, objname),
+                            "Please use {0}.{1} instead".format(modulename, objname))
+        if renames:
+            newname = renames[position]
+        else:
+            newname = objname
+        setattr(sys.modules[current], newname, newobj)
+deprecate_import(__name__, "bb.fetch", ("MalformedUrl", "encodeurl", "decodeurl"))
+deprecate_import(__name__, "bb.utils", ("mkdirhier", "movefile", "copyfile", "which"))
+deprecate_import(__name__, "bb.utils", ["vercmp_string"], ["vercmp"])
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 43dbfc1363..1f4107fb65 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -27,8 +27,9 @@
 from bb import data, event, mkdirhier, utils
 import bb, os, sys
+import bb.utils
-# When we execute a python function we'd like certain things 
+# When we execute a python function we'd like certain things
 # in all namespaces, hence we add them to __builtins__
 # If we do not do this and use the exec globals, they will
 # not be available to subfunctions.
@@ -98,18 +99,19 @@ def exec_func(func, d, dirs = None):
    ispython = flags['python']
-    cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
+    cleandirs = flags['cleandirs']
-    for cdir in cleandirs:
+    if cleandirs:
-        os.system("rm -rf %s" % cdir)
+        for cdir in data.expand(cleandirs, d).split():
+            os.system("rm -rf %s" % cdir)
-    if dirs:
+    if dirs is None:
-        dirs = data.expand(dirs, d)
+        dirs = flags['dirs']
-    else:
+        if dirs:
-        dirs = (data.expand(flags['dirs'], d) or "").split()
+            dirs = data.expand(dirs, d).split()
-    for adir in dirs:
-        mkdirhier(adir)
-    if len(dirs) > 0:
+    if dirs:
+        for adir in dirs:
+            bb.utils.mkdirhier(adir)
        adir = dirs[-1]
    else:
        adir = data.getVar('B', d, 1)
@@ -123,8 +125,8 @@ def exec_func(func, d, dirs = None):
    # Setup logfiles
    t = data.getVar('T', d, 1)
    if not t:
-        bb.msg.fatal(bb.msg.domain.Build, "T not set")
+        raise SystemExit("T variable not set, unable to build")
-    mkdirhier(t)
+    bb.utils.mkdirhier(t)
    logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
    runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
@@ -139,7 +141,7 @@ def exec_func(func, d, dirs = None):
            so = os.popen("tee \"%s\"" % logfile, "w")
        else:
            so = file(logfile, 'w')
-    except OSError, e:
+    except OSError as e:
        bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
        pass
@@ -156,9 +158,10 @@ def exec_func(func, d, dirs = None):
    os.dup2(se.fileno(), ose[1])
    locks = []
-    lockfiles = (data.expand(flags['lockfiles'], d) or "").split()
+    lockfiles = flags['lockfiles']
-    for lock in lockfiles:
+    if lockfiles:
-        locks.append(bb.utils.lockfile(lock))
+        for lock in data.expand(lockfiles, d).split():
+            locks.append(bb.utils.lockfile(lock))
    try:
        # Run the function
@@ -200,26 +203,22 @@ def exec_func(func, d, dirs = None):
 def exec_func_python(func, d, runfile, logfile):
    """Execute a python BB 'function'"""
-    import re, os
    bbfile = bb.data.getVar('FILE', d, 1)
-    tmp  = "def " + func + "():\n%s" % data.getVar(func, d)
+    tmp  = "def " + func + "(d):\n%s" % data.getVar(func, d)
-    tmp += '\n' + func + '()'
+    tmp += '\n' + func + '(d)'
    f = open(runfile, "w")
    f.write(tmp)
    comp = utils.better_compile(tmp, func, bbfile)
-    g = {} # globals
-    g['d'] = d
    try:
-        utils.better_exec(comp, g, tmp, bbfile)
+        utils.better_exec(comp, {"d": d}, tmp, bbfile)
    except:
-        (t,value,tb) = sys.exc_info()
+        (t, value, tb) = sys.exc_info()
        if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
            raise
-        bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func)
+        raise FuncFailed("Function %s failed" % func, logfile)
-        raise FuncFailed("function %s failed" % func, logfile)
 def exec_func_shell(func, d, runfile, logfile, flags):
    """Execute a shell BB 'function' Returns true if execution was successful.
@@ -248,7 +247,6 @@ def exec_func_shell(func, d, runfile, logfile, flags):
    f.close()
    os.chmod(runfile, 0775)
    if not func:
-        bb.msg.error(bb.msg.domain.Build, "Function not specified")
        raise FuncFailed("Function not specified for exec_func_shell")
    # execute function
@@ -262,7 +260,6 @@ def exec_func_shell(func, d, runfile, logfile, flags):
    if ret == 0:
        return
-    bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func)
    raise FuncFailed("function %s failed" % func, logfile)
@@ -287,7 +284,7 @@ def exec_task(task, d):
        event.fire(TaskStarted(task, localdata), localdata)
        exec_func(task, localdata)
        event.fire(TaskSucceeded(task, localdata), localdata)
-    except FuncFailed, message:
+    except FuncFailed as message:
        # Try to extract the optional logfile
        try:
            (msg, logfile) = message
@@ -305,8 +302,8 @@ def exec_task(task, d):
 def extract_stamp(d, fn):
    """
-    Extracts stamp format which is either a data dictonary (fn unset) 
+    Extracts stamp format which is either a data dictonary (fn unset)
-    or a dataCache entry (fn set). 
+    or a dataCache entry (fn set).
    """
    if fn:
        return d.stamp[fn]
@@ -323,7 +320,7 @@ def stamp_internal(task, d, file_name):
    if not stamp:
        return
    stamp = "%s.%s" % (stamp, task)
-    mkdirhier(os.path.dirname(stamp))
+    bb.utils.mkdirhier(os.path.dirname(stamp))
    # Remove the file and recreate to force timestamp
    # change on broken NFS filesystems
    if os.access(stamp, os.F_OK):
@@ -363,7 +360,7 @@ def add_tasks(tasklist, d):
        if not task in task_deps['tasks']:
            task_deps['tasks'].append(task)
-        flags = data.getVarFlags(task, d)    
+        flags = data.getVarFlags(task, d)
        def getTask(name):
            if not name in task_deps:
                task_deps[name] = {}
@@ -389,4 +386,3 @@ def remove_task(task, kill, d):
       If kill is 1, also remove tasks that depend on this task."""
    data.delVarFlag(task, 'task', d)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 1f180012e0..da4546640a 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -28,7 +28,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import os, re
+import os
 import bb.data
 import bb.utils
@@ -44,10 +44,10 @@ class Cache:
    """
    BitBake Cache implementation
    """
-    def __init__(self, cooker):
+    def __init__(self, data):
-        self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True)
+        self.cachedir = bb.data.getVar("CACHE", data, True)
        self.clean = {}
        self.checked = {}
        self.depends_cache = {}
@@ -61,30 +61,28 @@ class Cache:
            return
        self.has_cache = True
-        self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
+        self.cachefile = os.path.join(self.cachedir, "bb_cache.dat")
        bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
-        try:
+        bb.utils.mkdirhier(self.cachedir)
-            os.stat( self.cachedir )
-        except OSError:
-            bb.mkdirhier( self.cachedir )
        # If any of configuration.data's dependencies are newer than the
        # cache there isn't even any point in loading it...
        newest_mtime = 0
-        deps = bb.data.getVar("__depends", cooker.configuration.data, True)
+        deps = bb.data.getVar("__depends", data)
-        for f,old_mtime in deps:
-            if old_mtime > newest_mtime:
+        old_mtimes = [old_mtime for f, old_mtime in deps]
-                newest_mtime = old_mtime
+        old_mtimes.append(newest_mtime)
+        newest_mtime = max(old_mtimes)
        if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
            try:
                p = pickle.Unpickler(file(self.cachefile, "rb"))
                self.depends_cache, version_data = p.load()
                if version_data['CACHE_VER'] != __cache_version__:
-                    raise ValueError, 'Cache Version Mismatch'
+                    raise ValueError('Cache Version Mismatch')
                if version_data['BITBAKE_VER'] != bb.__version__:
-                    raise ValueError, 'Bitbake Version Mismatch'
+                    raise ValueError('Bitbake Version Mismatch')
            except EOFError:
                bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
                self.depends_cache = {}
@@ -92,27 +90,23 @@ class Cache:
                bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
                self.depends_cache = {}
        else:
-            try:
+            if os.path.isfile(self.cachefile):
-                os.stat( self.cachefile )
                bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...")
-            except OSError:
-                pass
    def getVar(self, var, fn, exp = 0):
        """
        Gets the value of a variable
        (similar to getVar in the data class)
-        
        There are two scenarios:
          1. We have cached data - serve from depends_cache[fn]
-          2. We're learning what data to cache - serve from data 
+          2. We're learning what data to cache - serve from data
             backend but add a copy of the data to the cache.
        """
        if fn in self.clean:
            return self.depends_cache[fn][var]
-        if not fn in self.depends_cache:
+        self.depends_cache.setdefault(fn, {})
-            self.depends_cache[fn] = {}
        if fn != self.data_fn:
            # We're trying to access data in the cache which doesn't exist
@@ -134,14 +128,14 @@ class Cache:
        self.data = data
        # Make sure __depends makes the depends_cache
-        # If we're a virtual class we need to make sure all our depends are appended 
+        # If we're a virtual class we need to make sure all our depends are appended
        # to the depends of fn.
-        depends = self.getVar("__depends", virtualfn, True) or []
+        depends = self.getVar("__depends", virtualfn) or set()
+        self.depends_cache.setdefault(fn, {})
        if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]:
            self.depends_cache[fn]["__depends"] = depends
-        for dep in depends:
+        else:
-            if dep not in self.depends_cache[fn]["__depends"]:
+            self.depends_cache[fn]["__depends"].update(depends)
-                self.depends_cache[fn]["__depends"].append(dep)
        # Make sure the variants always make it into the cache too
        self.getVar('__VARIANTS', virtualfn, True)
@@ -217,7 +211,7 @@ class Cache:
        for data in bb_data:
            virtualfn = self.realfn2virtual(fn, data)
            self.setData(virtualfn, fn, bb_data[data])
-            if self.getVar("__SKIPPED", virtualfn, True):
+            if self.getVar("__SKIPPED", virtualfn):
                skipped += 1
                bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn)
            else:
@@ -258,11 +252,11 @@ class Cache:
            self.remove(fn)
            return False
-        mtime = bb.parse.cached_mtime_noerror(fn) 
+        mtime = bb.parse.cached_mtime_noerror(fn)
        # Check file still exists
        if mtime == 0:
-            bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
+            bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn)
            self.remove(fn)
            return False
@@ -275,7 +269,7 @@ class Cache:
        # Check dependencies are still valid
        depends = self.getVar("__depends", fn, True)
        if depends:
-            for f,old_mtime in depends:
+            for f, old_mtime in depends:
                fmtime = bb.parse.cached_mtime_noerror(f)
                # Check if file still exists
                if old_mtime != 0 and fmtime == 0:
@@ -345,14 +339,14 @@ class Cache:
    def handle_data(self, file_name, cacheData):
        """
-        Save data we need into the cache 
+        Save data we need into the cache
        """
        pn       = self.getVar('PN', file_name, True)
        pe       = self.getVar('PE', file_name, True) or "0"
        pv       = self.getVar('PV', file_name, True)
        if 'SRCREVINACTION' in pv:
-            bb.note("Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name))
+            bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name))
        pr       = self.getVar('PR', file_name, True)
        dp       = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
        depends   = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
@@ -360,7 +354,7 @@ class Cache:
        packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
        rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
-        cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True)
+        cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name)
        # build PackageName to FileName lookup table
        if pn not in cacheData.pkg_pn:
@@ -371,7 +365,7 @@ class Cache:
        # build FileName to PackageName lookup table
        cacheData.pkg_fn[file_name] = pn
-        cacheData.pkg_pepvpr[file_name] = (pe,pv,pr)
+        cacheData.pkg_pepvpr[file_name] = (pe, pv, pr)
        cacheData.pkg_dp[file_name] = dp
        provides = [pn]
@@ -400,13 +394,13 @@ class Cache:
            if not dep in cacheData.all_depends:
                cacheData.all_depends.append(dep)
-        # Build reverse hash for PACKAGES, so runtime dependencies 
+        # Build reverse hash for PACKAGES, so runtime dependencies
        # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
        for package in packages:
            if not package in cacheData.packages:
                cacheData.packages[package] = []
            cacheData.packages[package].append(file_name)
-            rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() 
+            rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
        for package in packages_dynamic:
            if not package in cacheData.packages_dynamic:
@@ -445,38 +439,45 @@ class Cache:
        self.getVar('__BB_DONT_CACHE', file_name, True)
        self.getVar('__VARIANTS', file_name, True)
-    def load_bbfile( self, bbfile , config):
+    def load_bbfile( self, bbfile, config):
        """
        Load and parse one .bb build file
        Return the data and whether parsing resulted in the file being skipped
        """
+        chdir_back = False
-        import bb
+        from bb import data, parse
-        from bb import utils, data, parse, debug, event, fatal
        # expand tmpdir to include this topdir
        data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
        oldpath = os.path.abspath(os.getcwd())
-        if bb.parse.cached_mtime_noerror(bbfile_loc):
+        parse.cached_mtime_noerror(bbfile_loc)
-            os.chdir(bbfile_loc)
        bb_data = data.init_db(config)
+        # The ConfHandler first looks if there is a TOPDIR and if not
+        # then it would call getcwd().
+        # Previously, we chdir()ed to bbfile_loc, called the handler
+        # and finally chdir()ed back, a couple of thousand times. We now
+        # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
+        if not data.getVar('TOPDIR', bb_data):
+            chdir_back = True
+            data.setVar('TOPDIR', bbfile_loc, bb_data)
        try:
            bb_data = parse.handle(bbfile, bb_data) # read .bb data
-            os.chdir(oldpath)
+            if chdir_back: os.chdir(oldpath)
            return bb_data
        except:
-            os.chdir(oldpath)
+            if chdir_back: os.chdir(oldpath)
            raise
 def init(cooker):
    """
-    The Objective: Cache the minimum amount of data possible yet get to the 
+    The Objective: Cache the minimum amount of data possible yet get to the
    stage of building packages (i.e. tryBuild) without reparsing any .bb files.
-    To do this, we intercept getVar calls and only cache the variables we see 
+    To do this, we intercept getVar calls and only cache the variables we see
-    being accessed. We rely on the cache getVar calls being made for all 
+    being accessed. We rely on the cache getVar calls being made for all
-    variables bitbake might need to use to reach this stage. For each cached 
+    variables bitbake might need to use to reach this stage. For each cached
    file we need to track:
    * Its mtime
@@ -486,7 +487,7 @@ def init(cooker):
    Files causing parsing errors are evicted from the cache.
    """
-    return Cache(cooker)
+    return Cache(cooker.configuration.data)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index 06bd203c90..9a8d689e2a 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -20,7 +20,7 @@ Provide an interface to interact with the bitbake server through 'commands'
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 """
-The bitbake server takes 'commands' from its UI/commandline. 
+The bitbake server takes 'commands' from its UI/commandline.
 Commands are either synchronous or asynchronous.
 Async commands return data to the client in the form of events.
 Sync commands must only return data through the function return value
@@ -62,7 +62,7 @@ class Command:
        try:
            command = commandline.pop(0)
            if command in CommandsSync.__dict__:
-                # Can run synchronous commands straight away            
+                # Can run synchronous commands straight away
                return getattr(CommandsSync, command)(self.cmds_sync, self, commandline)
            if self.currentAsyncCommand is not None:
                return "Busy (%s in progress)" % self.currentAsyncCommand[0]
@@ -89,7 +89,17 @@ class Command:
                    return False
            else:
                return False
-        except:
+        except KeyboardInterrupt as exc:
+            self.finishAsyncCommand("Interrupted")
+            return False
+        except SystemExit as exc:
+            arg = exc.args[0]
+            if isinstance(arg, basestring):
+                self.finishAsyncCommand(arg)
+            else:
+                self.finishAsyncCommand("Exited with %s" % arg)
+            return False
+        except Exception:
            import traceback
            self.finishAsyncCommand(traceback.format_exc())
            return False
@@ -268,6 +278,3 @@ class CookerCommandSetExitCode(bb.event.Event):
    def  __init__(self, exitcode):
        bb.event.Event.__init__(self)
        self.exitcode = int(exitcode)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2406dfe95b..488bc610d2 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -22,11 +22,13 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import sys, os, getopt, glob, copy, os.path, re, time
+from __future__ import print_function
+import sys, os, glob, os.path, re, time
+import sre_constants
+from cStringIO import StringIO
+from contextlib import closing
 import bb
-from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
+from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
-from bb import command
-import itertools, sre_constants
 class MultipleMatches(Exception):
    """
@@ -121,11 +123,11 @@ class BBCooker:
            self.commandlineAction = None
            if 'world' in self.configuration.pkgs_to_build:
-                bb.error("'world' is not a valid target for --environment.")
+                bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.")
            elif len(self.configuration.pkgs_to_build) > 1:
-                bb.error("Only one target can be used with the --environment option.")
+                bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.")
            elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
-                bb.error("No target should be used with the --environment and --buildfile options.")
+                bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.")
            elif len(self.configuration.pkgs_to_build) > 0:
                self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
            else:
@@ -138,21 +140,18 @@ class BBCooker:
            self.commandlineAction = ["showVersions"]
        elif self.configuration.parse_only:
            self.commandlineAction = ["parseFiles"]
-        # FIXME - implement
-        #elif self.configuration.interactive:
-        #    self.interactiveMode()
        elif self.configuration.dot_graph:
            if self.configuration.pkgs_to_build:
                self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
            else:
                self.commandlineAction = None
-                bb.error("Please specify a package name for dependency graph generation.")
+                bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.")
        else:
            if self.configuration.pkgs_to_build:
                self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
            else:
                self.commandlineAction = None
-                bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+                bb.msg.error(bb.msg.domain.Build, "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
    def runCommands(self, server, data, abort):
        """
@@ -174,14 +173,14 @@ class BBCooker:
        except bb.build.FuncFailed:
            bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
            raise
-        except bb.build.EventException, e:
+        except bb.build.EventException as e:
            event = e.args[1]
            bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
            raise
    def tryBuild(self, fn, task):
        """
-        Build a provider and its dependencies. 
+        Build a provider and its dependencies.
        build_depends is a list of previous build dependencies (not runtime)
        If build_depends is empty, we're dealing with a runtime depends
        """
@@ -206,7 +205,7 @@ class BBCooker:
        # Sort by priority
        for pn in pkg_pn:
-            (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
+            (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
            preferred_versions[pn] = (pref_ver, pref_file)
            latest_versions[pn] = (last_ver, last_file)
@@ -260,27 +259,22 @@ class BBCooker:
        if fn:
            try:
                envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
-            except IOError, e:
+            except IOError as e:
                bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
                raise
-            except Exception, e:
+            except Exception as e:
                bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
                raise
-        class dummywrite:
-            def __init__(self):
-                self.writebuf = ""
-            def write(self, output):
-                self.writebuf = self.writebuf + output
        # emit variables and shell functions
        try:
            data.update_data(envdata)
-            wb = dummywrite()
+            with closing(StringIO()) as env:
-            data.emit_env(wb, envdata, True)
+                data.emit_env(env, envdata, True)
-            bb.msg.plain(wb.writebuf)
+                bb.msg.plain(env.getvalue())
-        except Exception, e:
+        except Exception as e:
            bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
        # emit the metadata which isnt valid shell
        data.expandKeys(envdata)
        for e in envdata.keys():
@@ -315,7 +309,7 @@ class BBCooker:
        rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
        rq.prepare_runqueue()
-        seen_fnids = []  
+        seen_fnids = []
        depend_tree = {}
        depend_tree["depends"] = {}
        depend_tree["tdepends"] = {}
@@ -352,7 +346,7 @@ class BBCooker:
                depend_tree["rdepends-pn"][pn] = []
                for rdep in taskdata.rdepids[fnid]:
-                        depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+                    depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
                rdepends = self.status.rundeps[fn]
                for package in rdepends:
@@ -397,51 +391,51 @@ class BBCooker:
        # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
        depends_file = file('pn-depends.dot', 'w' )
-        print >> depends_file, "digraph depends {"
+        print("digraph depends {", file=depends_file)
        for pn in depgraph["pn"]:
            fn = depgraph["pn"][pn]["filename"]
            version = depgraph["pn"][pn]["version"]
-            print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+            print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
        for pn in depgraph["depends"]:
            for depend in depgraph["depends"][pn]:
-                print >> depends_file, '"%s" -> "%s"' % (pn, depend)
+                print('"%s" -> "%s"' % (pn, depend), file=depends_file)
        for pn in depgraph["rdepends-pn"]:
            for rdepend in depgraph["rdepends-pn"][pn]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
+                print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
-        print >> depends_file,  "}"
+        print("}", file=depends_file)
        bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
        depends_file = file('package-depends.dot', 'w' )
-        print >> depends_file, "digraph depends {"
+        print("digraph depends {", file=depends_file)
        for package in depgraph["packages"]:
            pn = depgraph["packages"][package]["pn"]
            fn = depgraph["packages"][package]["filename"]
            version = depgraph["packages"][package]["version"]
            if package == pn:
-                print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
+                print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
            else:
-                print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
+                print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
            for depend in depgraph["depends"][pn]:
-                print >> depends_file, '"%s" -> "%s"' % (package, depend)
+                print('"%s" -> "%s"' % (package, depend), file=depends_file)
        for package in depgraph["rdepends-pkg"]:
            for rdepend in depgraph["rdepends-pkg"][package]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
+                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
        for package in depgraph["rrecs-pkg"]:
            for rdepend in depgraph["rrecs-pkg"][package]:
-                print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
+                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
-        print >> depends_file,  "}"
+        print("}", file=depends_file)
        bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
        tdepends_file = file('task-depends.dot', 'w' )
-        print >> tdepends_file, "digraph depends {"
+        print("digraph depends {", file=tdepends_file)
        for task in depgraph["tdepends"]:
            (pn, taskname) = task.rsplit(".", 1)
            fn = depgraph["pn"][pn]["filename"]
            version = depgraph["pn"][pn]["version"]
-            print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
+            print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
            for dep in depgraph["tdepends"][task]:
-                print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
+                print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
-        print >> tdepends_file,  "}"
+        print("}", file=tdepends_file)
        bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
    def buildDepgraph( self ):
@@ -452,9 +446,12 @@ class BBCooker:
        bb.data.update_data(localdata)
        bb.data.expandKeys(localdata)
+        matched = set()
        def calc_bbfile_priority(filename):
-            for (regex, pri) in self.status.bbfile_config_priorities:
+            for _, _, regex, pri in self.status.bbfile_config_priorities:
                if regex.match(filename):
+                    if not regex in matched:
+                        matched.add(regex)
                    return pri
            return 0
@@ -473,6 +470,11 @@ class BBCooker:
        for p in self.status.pkg_fn:
            self.status.bbfile_priority[p] = calc_bbfile_priority(p)
+        for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
+            if not regex in matched:
+                bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" %
+                                                    (collection, pattern))
    def buildWorldTargetList(self):
        """
         Build package list for "bitbake world"
@@ -505,31 +507,57 @@ class BBCooker:
        """Drop off into a shell"""
        try:
            from bb import shell
-        except ImportError, details:
+        except ImportError as details:
            bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
        else:
            shell.start( self )
+    def _findLayerConf(self):
+        path = os.getcwd()
+        while path != "/":
+            bblayers = os.path.join(path, "conf", "bblayers.conf")
+            if os.path.exists(bblayers):
+                return bblayers
+            path, _ = os.path.split(path)
    def parseConfigurationFiles(self, files):
        try:
            data = self.configuration.data
            for f in files:
                data = bb.parse.handle(f, data)
-            layerconf = os.path.join(os.getcwd(), "conf", "bblayers.conf")
+            layerconf = self._findLayerConf()
-            if os.path.exists(layerconf):
+            if layerconf:
                bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
                data = bb.parse.handle(layerconf, data)
                layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
+                data = bb.data.createCopy(data)
                for layer in layers:
                    bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
                    bb.data.setVar('LAYERDIR', layer, data)
                    data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
+                    # XXX: Hack, relies on the local keys of the datasmart
+                    # instance being stored in the 'dict' attribute and makes
+                    # assumptions about how variable expansion works, but
+                    # there's no better way to force an expansion of a single
+                    # variable across the datastore today, and this at least
+                    # lets us reference LAYERDIR without having to immediately
+                    # eval all our variables that use it.
+                    for key in data.dict:
+                        if key != "_data":
+                            value = data.getVar(key, False)
+                            if value and "${LAYERDIR}" in value:
+                                data.setVar(key, value.replace("${LAYERDIR}", layer))
                bb.data.delVar('LAYERDIR', data)
+            if not data.getVar("BBPATH", True):
+                bb.fatal("The BBPATH variable is not set")
            data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
            self.configuration.data = data
@@ -541,16 +569,17 @@ class BBCooker:
            # Nomally we only register event handlers at the end of parsing .bb files
            # We register any handlers we've found so far here...
-            for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
+            for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []:
-                bb.event.register(var,bb.data.getVar(var, self.configuration.data))
+                bb.event.register(var, bb.data.getVar(var, self.configuration.data))
            bb.fetch.fetcher_init(self.configuration.data)
            bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
-        except IOError, e:
+        except IOError as e:
            bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
-        except bb.parse.ParseError, details:
+        except bb.parse.ParseError as details:
            bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
    def handleCollections( self, collections ):
@@ -573,7 +602,7 @@ class BBCooker:
                    continue
                try:
                    pri = int(priority)
-                    self.status.bbfile_config_priorities.append((cre, pri))
+                    self.status.bbfile_config_priorities.append((c, regex, cre, pri))
                except ValueError:
                    bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
@@ -582,8 +611,8 @@ class BBCooker:
        Setup any variables needed before starting a build
        """
        if not bb.data.getVar("BUILDNAME", self.configuration.data):
-            bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
+            bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data)
-        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
+        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data)
    def matchFiles(self, buildfile):
        """
@@ -630,13 +659,19 @@ class BBCooker:
        if (task == None):
            task = self.configuration.cmd
-        fn = self.matchFile(buildfile)
+        self.bb_cache = bb.cache.init(self)
+        self.status = bb.cache.CacheData()
+        (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
+        buildfile = self.matchFile(fn)
+        fn = self.bb_cache.realfn2virtual(buildfile, cls)
        self.buildSetVars()
        # Load data into the cache for fn and parse the loaded cache data
-        self.bb_cache = bb.cache.init(self)
+        the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
-        self.status = bb.cache.CacheData()
+        self.bb_cache.setData(fn, buildfile, the_data)
-        self.bb_cache.loadData(fn, self.configuration.data, self.status)      
+        self.bb_cache.handle_data(fn, self.status)
        # Tweak some variables
        item = self.bb_cache.getVar('PN', fn, True)
@@ -675,8 +710,8 @@ class BBCooker:
            failures = 0
            try:
                retval = rq.execute_runqueue()
-            except runqueue.TaskFailure, fnids:
+            except runqueue.TaskFailure as exc:
-                for fnid in fnids:
+                for fnid in exc.args:
                    bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
                    failures = failures + 1
                retval = False
@@ -711,8 +746,8 @@ class BBCooker:
            failures = 0
            try:
                retval = rq.execute_runqueue()
-            except runqueue.TaskFailure, fnids:
+            except runqueue.TaskFailure as exc:
-                for fnid in fnids:
+                for fnid in exc.args:
                    bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
                    failures = failures + 1
                retval = False
@@ -769,10 +804,10 @@ class BBCooker:
            ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
            self.status.ignored_dependencies = set(ignore.split())
-    
            for dep in self.configuration.extra_assume_provided:
                self.status.ignored_dependencies.add(dep)
-    
            self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
            bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
@@ -810,7 +845,7 @@ class BBCooker:
        for f in contents:
            (root, ext) = os.path.splitext(f)
            if ext == ".bb":
-                bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
+                bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
        return bbfiles
    def find_bbfiles( self, path ):
@@ -822,7 +857,7 @@ class BBCooker:
            for ignored in ('SCCS', 'CVS', '.svn'):
                if ignored in dirs:
                    dirs.remove(ignored)
-            found += [join(dir,f) for f in files if f.endswith('.bb')]
+            found += [join(dir, f) for f in files if f.endswith('.bb')]
        return found
@@ -906,9 +941,9 @@ class BBCooker:
            pout.close()
        else:
            self.server.serve_forever()
-        
        bb.event.fire(CookerExit(), self.configuration.event_data)
-        
 class CookerExit(bb.event.Event):
    """
    Notify clients of the Cooker shutdown
@@ -937,9 +972,9 @@ class CookerParser:
        self.pointer = 0
    def parse_next(self):
+        cooker = self.cooker
        if self.pointer < len(self.filelist):
            f = self.filelist[self.pointer]
-            cooker = self.cooker
            try:
                fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
@@ -951,7 +986,7 @@ class CookerParser:
                self.skipped += skipped
                self.virtuals += virtuals
-            except IOError, e:
+            except IOError as e:
                self.error += 1
                cooker.bb_cache.remove(f)
                bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
@@ -960,7 +995,7 @@ class CookerParser:
                cooker.bb_cache.remove(f)
                cooker.bb_cache.sync()
                raise
-            except Exception, e:
+            except Exception as e:
                self.error += 1
                cooker.bb_cache.remove(f)
                bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
@@ -978,4 +1013,3 @@ class CookerParser:
                raise ParsingErrorsFound
            return False
        return True
diff --git a/bitbake/lib/bb/daemonize.py b/bitbake/lib/bb/daemonize.py
index 1a8bb379f4..f0714b3af6 100644
--- a/bitbake/lib/bb/daemonize.py
+++ b/bitbake/lib/bb/daemonize.py
@@ -1,191 +1,190 @@
-"""
+"""
-Python Deamonizing helper
+Python Deamonizing helper
-Configurable daemon behaviors:
+Configurable daemon behaviors:
-   1.) The current working directory set to the "/" directory.
+    1.) The current working directory set to the "/" directory.
-   2.) The current file creation mode mask set to 0.
+    2.) The current file creation mode mask set to 0.
-   3.) Close all open files (1024). 
+    3.) Close all open files (1024). 
-   4.) Redirect standard I/O streams to "/dev/null".
+    4.) Redirect standard I/O streams to "/dev/null".
-A failed call to fork() now raises an exception.
+A failed call to fork() now raises an exception.
-References:
+References:
-   1) Advanced Programming in the Unix Environment: W. Richard Stevens
+    1) Advanced Programming in the Unix Environment: W. Richard Stevens
-   2) Unix Programming Frequently Asked Questions:
+    2) Unix Programming Frequently Asked Questions:
-         http://www.erlenstar.demon.co.uk/unix/faq_toc.html
+            http://www.erlenstar.demon.co.uk/unix/faq_toc.html
-Modified to allow a function to be daemonized and return for 
+Modified to allow a function to be daemonized and return for 
-bitbake use by Richard Purdie
+bitbake use by Richard Purdie
-"""
+"""
-__author__ = "Chad J. Schroeder"
+__author__ = "Chad J. Schroeder"
-__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
+__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
-__version__ = "0.2"
+__version__ = "0.2"
-# Standard Python modules.
+# Standard Python modules.
-import os               # Miscellaneous OS interfaces.
+import os                    # Miscellaneous OS interfaces.
-import sys              # System-specific parameters and functions.
+import sys                  # System-specific parameters and functions.
-# Default daemon parameters.
+# Default daemon parameters.
-# File mode creation mask of the daemon.
+# File mode creation mask of the daemon.
-# For BitBake's children, we do want to inherit the parent umask.
+# For BitBake's children, we do want to inherit the parent umask.
-UMASK = None
+UMASK = None
-# Default maximum for the number of available file descriptors.
+# Default maximum for the number of available file descriptors.
-MAXFD = 1024
+MAXFD = 1024
-# The standard I/O file descriptors are redirected to /dev/null by default.
+# The standard I/O file descriptors are redirected to /dev/null by default.
-if (hasattr(os, "devnull")):
+if (hasattr(os, "devnull")):
-   REDIRECT_TO = os.devnull
+    REDIRECT_TO = os.devnull
-else:
+else:
-   REDIRECT_TO = "/dev/null"
+    REDIRECT_TO = "/dev/null"
-def createDaemon(function, logfile):
+def createDaemon(function, logfile):
-   """
+    """
-   Detach a process from the controlling terminal and run it in the
+    Detach a process from the controlling terminal and run it in the
-   background as a daemon, returning control to the caller.
+    background as a daemon, returning control to the caller.
-   """
+    """
-   try:
+    try:
-      # Fork a child process so the parent can exit.  This returns control to
+        # Fork a child process so the parent can exit.  This returns control to
-      # the command-line or shell.  It also guarantees that the child will not
+        # the command-line or shell.  It also guarantees that the child will not
-      # be a process group leader, since the child receives a new process ID
+        # be a process group leader, since the child receives a new process ID
-      # and inherits the parent's process group ID.  This step is required
+        # and inherits the parent's process group ID.  This step is required
-      # to insure that the next call to os.setsid is successful.
+        # to insure that the next call to os.setsid is successful.
-      pid = os.fork()
+        pid = os.fork()
-   except OSError, e:
+    except OSError as e:
-      raise Exception, "%s [%d]" % (e.strerror, e.errno)
+        raise Exception("%s [%d]" % (e.strerror, e.errno))
-   if (pid == 0):       # The first child.
+    if (pid == 0):      # The first child.
-      # To become the session leader of this new session and the process group
+        # To become the session leader of this new session and the process group
-      # leader of the new process group, we call os.setsid().  The process is
+        # leader of the new process group, we call os.setsid().  The process is
-      # also guaranteed not to have a controlling terminal.
+        # also guaranteed not to have a controlling terminal.
-      os.setsid()
+        os.setsid()
-      # Is ignoring SIGHUP necessary?
+        # Is ignoring SIGHUP necessary?
-      #
+        #
-      # It's often suggested that the SIGHUP signal should be ignored before
+        # It's often suggested that the SIGHUP signal should be ignored before
-      # the second fork to avoid premature termination of the process.  The
+        # the second fork to avoid premature termination of the process.  The
-      # reason is that when the first child terminates, all processes, e.g.
+        # reason is that when the first child terminates, all processes, e.g.
-      # the second child, in the orphaned group will be sent a SIGHUP.
+        # the second child, in the orphaned group will be sent a SIGHUP.
-      #
+        #
-      # "However, as part of the session management system, there are exactly
+        # "However, as part of the session management system, there are exactly
-      # two cases where SIGHUP is sent on the death of a process:
+        # two cases where SIGHUP is sent on the death of a process:
-      #
+        #
-      #   1) When the process that dies is the session leader of a session that
+        #    1) When the process that dies is the session leader of a session that
-      #      is attached to a terminal device, SIGHUP is sent to all processes
+        #        is attached to a terminal device, SIGHUP is sent to all processes
-      #      in the foreground process group of that terminal device.
+        #        in the foreground process group of that terminal device.
-      #   2) When the death of a process causes a process group to become
+        #    2) When the death of a process causes a process group to become
-      #      orphaned, and one or more processes in the orphaned group are
+        #        orphaned, and one or more processes in the orphaned group are
-      #      stopped, then SIGHUP and SIGCONT are sent to all members of the
+        #        stopped, then SIGHUP and SIGCONT are sent to all members of the
-      #      orphaned group." [2]
+        #        orphaned group." [2]
-      #
+        #
-      # The first case can be ignored since the child is guaranteed not to have
+        # The first case can be ignored since the child is guaranteed not to have
-      # a controlling terminal.  The second case isn't so easy to dismiss.
+        # a controlling terminal.  The second case isn't so easy to dismiss.
-      # The process group is orphaned when the first child terminates and
+        # The process group is orphaned when the first child terminates and
-      # POSIX.1 requires that every STOPPED process in an orphaned process
+        # POSIX.1 requires that every STOPPED process in an orphaned process
-      # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
+        # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
-      # second child is not STOPPED though, we can safely forego ignoring the
+        # second child is not STOPPED though, we can safely forego ignoring the
-      # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
+        # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
-      #
+        #
-      # import signal           # Set handlers for asynchronous events.
+        # import signal              # Set handlers for asynchronous events.
-      # signal.signal(signal.SIGHUP, signal.SIG_IGN)
+        # signal.signal(signal.SIGHUP, signal.SIG_IGN)
-      try:
+        try:
-         # Fork a second child and exit immediately to prevent zombies.  This
+            # Fork a second child and exit immediately to prevent zombies.  This
-         # causes the second child process to be orphaned, making the init
+            # causes the second child process to be orphaned, making the init
-         # process responsible for its cleanup.  And, since the first child is
+            # process responsible for its cleanup.  And, since the first child is
-         # a session leader without a controlling terminal, it's possible for
+            # a session leader without a controlling terminal, it's possible for
-         # it to acquire one by opening a terminal in the future (System V-
+            # it to acquire one by opening a terminal in the future (System V-
-         # based systems).  This second fork guarantees that the child is no
+            # based systems).  This second fork guarantees that the child is no
-         # longer a session leader, preventing the daemon from ever acquiring
+            # longer a session leader, preventing the daemon from ever acquiring
-         # a controlling terminal.
+            # a controlling terminal.
-         pid = os.fork()        # Fork a second child.
+            pid = os.fork()     # Fork a second child.
-      except OSError, e:
+        except OSError as e:
-         raise Exception, "%s [%d]" % (e.strerror, e.errno)
+            raise Exception("%s [%d]" % (e.strerror, e.errno))
-      if (pid == 0):    # The second child.
+        if (pid == 0):  # The second child.
-         # We probably don't want the file mode creation mask inherited from
+            # We probably don't want the file mode creation mask inherited from
-         # the parent, so we give the child complete control over permissions.
+            # the parent, so we give the child complete control over permissions.
-         if UMASK is not None:
+            if UMASK is not None:
-             os.umask(UMASK)
+                os.umask(UMASK)
-      else:
+        else:
-         # Parent (the first child) of the second child.
+            # Parent (the first child) of the second child.
-         os._exit(0)
+            os._exit(0)
-   else:
+    else:
-      # exit() or _exit()?
+        # exit() or _exit()?
-      # _exit is like exit(), but it doesn't call any functions registered
+        # _exit is like exit(), but it doesn't call any functions registered
-      # with atexit (and on_exit) or any registered signal handlers.  It also
+        # with atexit (and on_exit) or any registered signal handlers.  It also
-      # closes any open file descriptors.  Using exit() may cause all stdio
+        # closes any open file descriptors.  Using exit() may cause all stdio
-      # streams to be flushed twice and any temporary files may be unexpectedly
+        # streams to be flushed twice and any temporary files may be unexpectedly
-      # removed.  It's therefore recommended that child branches of a fork()
+        # removed.  It's therefore recommended that child branches of a fork()
-      # and the parent branch(es) of a daemon use _exit().
+        # and the parent branch(es) of a daemon use _exit().
-      return
+        return
-   # Close all open file descriptors.  This prevents the child from keeping
+    # Close all open file descriptors.  This prevents the child from keeping
-   # open any file descriptors inherited from the parent.  There is a variety
+    # open any file descriptors inherited from the parent.  There is a variety
-   # of methods to accomplish this task.  Three are listed below.
+    # of methods to accomplish this task.  Three are listed below.
-   #
+    #
-   # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
+    # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
-   # number of open file descriptors to close.  If it doesn't exists, use
+    # number of open file descriptors to close.  If it doesn't exists, use
-   # the default value (configurable).
+    # the default value (configurable).
-   #
+    #
-   # try:
+    # try:
-   #    maxfd = os.sysconf("SC_OPEN_MAX")
+    #     maxfd = os.sysconf("SC_OPEN_MAX")
-   # except (AttributeError, ValueError):
+    # except (AttributeError, ValueError):
-   #    maxfd = MAXFD
+    #     maxfd = MAXFD
-   #
+    #
-   # OR
+    # OR
-   #
+    #
-   # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
+    # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
-   #    maxfd = os.sysconf("SC_OPEN_MAX")
+    #     maxfd = os.sysconf("SC_OPEN_MAX")
-   # else:
+    # else:
-   #    maxfd = MAXFD
+    #     maxfd = MAXFD
-   #
+    #
-   # OR
+    # OR
-   #
+    #
-   # Use the getrlimit method to retrieve the maximum file descriptor number
+    # Use the getrlimit method to retrieve the maximum file descriptor number
-   # that can be opened by this process.  If there is not limit on the
+    # that can be opened by this process.  If there is not limit on the
-   # resource, use the default value.
+    # resource, use the default value.
-   #
+    #
-   import resource              # Resource usage information.
+    import resource             # Resource usage information.
-   maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-   if (maxfd == resource.RLIM_INFINITY):
+    if (maxfd == resource.RLIM_INFINITY):
-      maxfd = MAXFD
+        maxfd = MAXFD
-  
+  
-   # Iterate through and close all file descriptors.
+    # Iterate through and close all file descriptors.
-#   for fd in range(0, maxfd):
+#    for fd in range(0, maxfd):
-#      try:
+#        try:
-#         os.close(fd)
+#            os.close(fd)
-#      except OSError:  # ERROR, fd wasn't open to begin with (ignored)
+#        except OSError:        # ERROR, fd wasn't open to begin with (ignored)
-#         pass
+#            pass
-   # Redirect the standard I/O file descriptors to the specified file.  Since
+    # Redirect the standard I/O file descriptors to the specified file.  Since
-   # the daemon has no controlling terminal, most daemons redirect stdin,
+    # the daemon has no controlling terminal, most daemons redirect stdin,
-   # stdout, and stderr to /dev/null.  This is done to prevent side-effects
+    # stdout, and stderr to /dev/null.  This is done to prevent side-effects
-   # from reads and writes to the standard I/O file descriptors.
+    # from reads and writes to the standard I/O file descriptors.
-   # This call to open is guaranteed to return the lowest file descriptor,
+    # This call to open is guaranteed to return the lowest file descriptor,
-   # which will be 0 (stdin), since it was closed above.
+    # which will be 0 (stdin), since it was closed above.
-#   os.open(REDIRECT_TO, os.O_RDWR)     # standard input (0)
+#    os.open(REDIRECT_TO, os.O_RDWR)    # standard input (0)
-   # Duplicate standard input to standard output and standard error.
+    # Duplicate standard input to standard output and standard error.
-#   os.dup2(0, 1)                       # standard output (1)
+#    os.dup2(0, 1)                      # standard output (1)
-#   os.dup2(0, 2)                       # standard error (2)
+#    os.dup2(0, 2)                      # standard error (2)
-   si = file('/dev/null', 'r')
+    si = file('/dev/null', 'r')
-   so = file(logfile, 'w')
+    so = file(logfile, 'w')
-   se = so
+    se = so
-   # Replace those fds with our own
+    # Replace those fds with our own
-   os.dup2(si.fileno(), sys.stdin.fileno())
+    os.dup2(si.fileno(), sys.stdin.fileno())
-   os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(so.fileno(), sys.stdout.fileno())
-   os.dup2(se.fileno(), sys.stderr.fileno())
+    os.dup2(se.fileno(), sys.stderr.fileno())
-   function()
+    function()
-   os._exit(0)
+    os._exit(0)
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index c3bb1a1f43..636983edcc 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -11,7 +11,7 @@ operations. At night the cookie monster came by and
 suggested 'give me cookies on setting the variables and
 things will work out'. Taking this suggestion into account
 applying the skills from the not yet passed 'Entwurf und
-Analyse von Algorithmen' lecture and the cookie 
+Analyse von Algorithmen' lecture and the cookie
 monster seems to be right. We will track setVar more carefully
 to have faster update_data and expandKeys operations.
@@ -37,39 +37,40 @@ the speed is more critical here.
 #
 #Based on functions from the base bb module, Copyright 2003 Holger Schurig
-import sys, os, re, types
+import sys, os, re
 if sys.argv[0][-5:] == "pydoc":
    path = os.path.dirname(os.path.dirname(sys.argv[1]))
 else:
    path = os.path.dirname(os.path.dirname(sys.argv[0]))
-sys.path.insert(0,path)
+sys.path.insert(0, path)
+from itertools import groupby
 from bb import data_smart
 import bb
-class VarExpandError(Exception):
-    pass
 _dict_type = data_smart.DataSmart
 def init():
+    """Return a new object representing the Bitbake data"""
    return _dict_type()
 def init_db(parent = None):
+    """Return a new object representing the Bitbake data,
+    optionally based on an existing object"""
    if parent:
        return parent.createCopy()
    else:
        return _dict_type()
 def createCopy(source):
-     """Link the source set to the destination
+    """Link the source set to the destination
-     If one does not find the value in the destination set,
+    If one does not find the value in the destination set,
-     search will go on to the source set to get the value.
+    search will go on to the source set to get the value.
-     Value from source are copy-on-write. i.e. any try to
+    Value from source are copy-on-write. i.e. any try to
-     modify one of them will end up putting the modified value
+    modify one of them will end up putting the modified value
-     in the destination set.
+    in the destination set.
-     """
+    """
-     return source.createCopy()
+    return source.createCopy()
 def initVar(var, d):
    """Non-destructive var init for data structure"""
@@ -77,91 +78,34 @@ def initVar(var, d):
 def setVar(var, value, d):
-    """Set a variable to a given value
+    """Set a variable to a given value"""
+    d.setVar(var, value)
-    Example:
-        >>> d = init()
-        >>> setVar('TEST', 'testcontents', d)
-        >>> print getVar('TEST', d)
-        testcontents
-    """
-    d.setVar(var,value)
 def getVar(var, d, exp = 0):
-    """Gets the value of a variable
+    """Gets the value of a variable"""
+    return d.getVar(var, exp)
-    Example:
-        >>> d = init()
-        >>> setVar('TEST', 'testcontents', d)
-        >>> print getVar('TEST', d)
-        testcontents
-    """
-    return d.getVar(var,exp)
 def renameVar(key, newkey, d):
-    """Renames a variable from key to newkey
+    """Renames a variable from key to newkey"""
-    Example:
-        >>> d = init()
-        >>> setVar('TEST', 'testcontents', d)
-        >>> renameVar('TEST', 'TEST2', d)
-        >>> print getVar('TEST2', d)
-        testcontents
-    """
    d.renameVar(key, newkey)
 def delVar(var, d):
-    """Removes a variable from the data set
+    """Removes a variable from the data set"""
-    Example:
-        >>> d = init()
-        >>> setVar('TEST', 'testcontents', d)
-        >>> print getVar('TEST', d)
-        testcontents
-        >>> delVar('TEST', d)
-        >>> print getVar('TEST', d)
-        None
-    """
    d.delVar(var)
 def setVarFlag(var, flag, flagvalue, d):
-    """Set a flag for a given variable to a given value
+    """Set a flag for a given variable to a given value"""
+    d.setVarFlag(var, flag, flagvalue)
-    Example:
-        >>> d = init()
-        >>> setVarFlag('TEST', 'python', 1, d)
-        >>> print getVarFlag('TEST', 'python', d)
-        1
-    """
-    d.setVarFlag(var,flag,flagvalue)
 def getVarFlag(var, flag, d):
-    """Gets given flag from given var
+    """Gets given flag from given var"""
+    return d.getVarFlag(var, flag)
-    Example:
-        >>> d = init()
-        >>> setVarFlag('TEST', 'python', 1, d)
-        >>> print getVarFlag('TEST', 'python', d)
-        1
-    """
-    return d.getVarFlag(var,flag)
 def delVarFlag(var, flag, d):
-    """Removes a given flag from the variable's flags
+    """Removes a given flag from the variable's flags"""
+    d.delVarFlag(var, flag)
-    Example:
-        >>> d = init()
-        >>> setVarFlag('TEST', 'testflag', 1, d)
-        >>> print getVarFlag('TEST', 'testflag', d)
-        1
-        >>> delVarFlag('TEST', 'testflag', d)
-        >>> print getVarFlag('TEST', 'testflag', d)
-        None
-    """
-    d.delVarFlag(var,flag)
 def setVarFlags(var, flags, d):
    """Set the flags for a given variable
@@ -170,115 +114,27 @@ def setVarFlags(var, flags, d):
        setVarFlags will not clear previous
        flags. Think of this method as
        addVarFlags
-    Example:
-        >>> d = init()
-        >>> myflags = {}
-        >>> myflags['test'] = 'blah'
-        >>> setVarFlags('TEST', myflags, d)
-        >>> print getVarFlag('TEST', 'test', d)
-        blah
    """
-    d.setVarFlags(var,flags)
+    d.setVarFlags(var, flags)
 def getVarFlags(var, d):
-    """Gets a variable's flags
+    """Gets a variable's flags"""
-    Example:
-        >>> d = init()
-        >>> setVarFlag('TEST', 'test', 'blah', d)
-        >>> print getVarFlags('TEST', d)['test']
-        blah
-    """
    return d.getVarFlags(var)
 def delVarFlags(var, d):
-    """Removes a variable's flags
+    """Removes a variable's flags"""
-    Example:
-        >>> data = init()
-        >>> setVarFlag('TEST', 'testflag', 1, data)
-        >>> print getVarFlag('TEST', 'testflag', data)
-        1
-        >>> delVarFlags('TEST', data)
-        >>> print getVarFlags('TEST', data)
-        None
-    """
    d.delVarFlags(var)
 def keys(d):
-    """Return a list of keys in d
+    """Return a list of keys in d"""
-    Example:
-        >>> d = init()
-        >>> setVar('TEST',  1, d)
-        >>> setVar('MOO' ,  2, d)
-        >>> setVarFlag('TEST', 'test', 1, d)
-        >>> keys(d)
-        ['TEST', 'MOO']
-    """
    return d.keys()
-def getData(d):
-    """Returns the data object used"""
-    return d
-def setData(newData, d):
-    """Sets the data object to the supplied value"""
-    d = newData
-##
-## Cookie Monsters' query functions
-##
-def _get_override_vars(d, override):
-    """
-    Internal!!!
-    Get the Names of Variables that have a specific
-    override. This function returns a iterable
-    Set or an empty list
-    """
-    return []
-def _get_var_flags_triple(d):
-    """
-    Internal!!!
-    """
-    return []
 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 def expand(s, d, varname = None):
-    """Variable expansion using the data store.
+    """Variable expansion using the data store"""
-    Example:
-        Standard expansion:
-        >>> d = init()
-        >>> setVar('A', 'sshd', d)
-        >>> print expand('/usr/bin/${A}', d)
-        /usr/bin/sshd
-        Python expansion:
-        >>> d = init()
-        >>> print expand('result: ${@37 * 72}', d)
-        result: 2664
-        Shell expansion:
-        >>> d = init()
-        >>> print expand('${TARGET_MOO}', d)
-        ${TARGET_MOO}
-        >>> setVar('TARGET_MOO', 'yupp', d)
-        >>> print expand('${TARGET_MOO}',d)
-        yupp
-        >>> setVar('SRC_URI', 'http://somebug.${TARGET_MOO}', d)
-        >>> delVar('TARGET_MOO', d)
-        >>> print expand('${SRC_URI}', d)
-        http://somebug.${TARGET_MOO}
-    """
    return d.expand(s, varname)
 def expandKeys(alterdata, readdata = None):
@@ -295,38 +151,13 @@ def expandKeys(alterdata, readdata = None):
            continue
        todolist[key] = ekey
-    # These two for loops are split for performance to maximise the 
+    # These two for loops are split for performance to maximise the
    # usefulness of the expand cache
    for key in todolist:
        ekey = todolist[key]
        renameVar(key, ekey, alterdata)
-def expandData(alterdata, readdata = None):
-    """For each variable in alterdata, expand it, and update the var contents.
-       Replacements use data from readdata.
-    Example:
-        >>> a=init()
-        >>> b=init()
-        >>> setVar("dlmsg", "dl_dir is ${DL_DIR}", a)
-        >>> setVar("DL_DIR", "/path/to/whatever", b)
-        >>> expandData(a, b)
-        >>> print getVar("dlmsg", a)
-        dl_dir is /path/to/whatever
-       """
-    if readdata == None:
-        readdata = alterdata
-    for key in keys(alterdata):
-        val = getVar(key, alterdata)
-        if type(val) is not types.StringType:
-            continue
-        expanded = expand(val, readdata)
-#       print "key is %s, val is %s, expanded is %s" % (key, val, expanded)
-        if val != expanded:
-            setVar(key, expanded, alterdata)
 def inheritFromOS(d):
    """Inherit variables from the environment."""
    for s in os.environ.keys():
@@ -351,21 +182,15 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
        if all:
            oval = getVar(var, d, 0)
        val = getVar(var, d, 1)
-    except KeyboardInterrupt:
+    except (KeyboardInterrupt, bb.build.FuncFailed):
        raise
-    except:
+    except Exception, exc:
-        excname = str(sys.exc_info()[0])
+        o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
-        if excname == "bb.build.FuncFailed":
-            raise
-        o.write('# expansion of %s threw %s\n' % (var, excname))
        return 0
    if all:
        o.write('# %s=%s\n' % (var, oval))
-    if type(val) is not types.StringType:
-        return 0
    if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
        return 0
@@ -375,10 +200,11 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
        o.write('unset %s\n' % varExpanded)
        return 1
-    val.rstrip()
    if not val:
        return 0
+    val = str(val)
    if func:
        # NOTE: should probably check for unbalanced {} within the var
        o.write("%s() {\n%s\n}\n" % (varExpanded, val))
@@ -393,173 +219,22 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
    o.write('%s="%s"\n' % (varExpanded, alter))
    return 1
 def emit_env(o=sys.__stdout__, d = init(), all=False):
    """Emits all items in the data store in a format such that it can be sourced by a shell."""
-    env = keys(d)
+    isfunc = lambda key: bool(d.getVarFlag(key, "func"))
+    keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
-    for e in env:
+    grouped = groupby(keys, isfunc)
-        if getVarFlag(e, "func", d):
+    for isfunc, keys in grouped:
-            continue
+        for key in keys:
-        emit_var(e, o, d, all) and o.write('\n')
+            emit_var(key, o, d, all and not isfunc) and o.write('\n')
-    for e in env:
-        if not getVarFlag(e, "func", d):
-            continue
-        emit_var(e, o, d) and o.write('\n')
 def update_data(d):
-    """Modifies the environment vars according to local overrides and commands.
+    """Performs final steps upon the datastore, including application of overrides"""
-    Examples:
+    d.finalize()
-        Appending to a variable:
-        >>> d = init()
-        >>> setVar('TEST', 'this is a', d)
-        >>> setVar('TEST_append', ' test', d)
-        >>> setVar('TEST_append', ' of the emergency broadcast system.', d)
-        >>> update_data(d)
-        >>> print getVar('TEST', d)
-        this is a test of the emergency broadcast system.
-        Prepending to a variable:
-        >>> setVar('TEST', 'virtual/libc', d)
-        >>> setVar('TEST_prepend', 'virtual/tmake ', d)
-        >>> setVar('TEST_prepend', 'virtual/patcher ', d)
-        >>> update_data(d)
-        >>> print getVar('TEST', d)
-        virtual/patcher virtual/tmake virtual/libc
-        Overrides:
-        >>> setVar('TEST_arm', 'target', d)
-        >>> setVar('TEST_ramses', 'machine', d)
-        >>> setVar('TEST_local', 'local', d)
-        >>> setVar('OVERRIDES', 'arm', d)
-        >>> setVar('TEST', 'original', d)
-        >>> update_data(d)
-        >>> print getVar('TEST', d)
-        target
-        >>> setVar('OVERRIDES', 'arm:ramses:local', d)
-        >>> setVar('TEST', 'original', d)
-        >>> update_data(d)
-        >>> print getVar('TEST', d)
-        local
-        CopyMonster:
-        >>> e = d.createCopy()
-        >>> setVar('TEST_foo', 'foo', e)
-        >>> update_data(e)
-        >>> print getVar('TEST', e)
-        local
-        >>> setVar('OVERRIDES', 'arm:ramses:local:foo', e)
-        >>> update_data(e)
-        >>> print getVar('TEST', e)
-        foo
-        >>> f = d.createCopy()
-        >>> setVar('TEST_moo', 'something', f)
-        >>> setVar('OVERRIDES', 'moo:arm:ramses:local:foo', e)
-        >>> update_data(e)
-        >>> print getVar('TEST', e)
-        foo
-        >>> h = init()
-        >>> setVar('SRC_URI', 'file://append.foo;patch=1 ', h)
-        >>> g = h.createCopy()
-        >>> setVar('SRC_URI_append_arm', 'file://other.foo;patch=1', g)
-        >>> setVar('OVERRIDES', 'arm:moo', g)
-        >>> update_data(g)
-        >>> print getVar('SRC_URI', g)
-        file://append.foo;patch=1 file://other.foo;patch=1
-    """
-    bb.msg.debug(2, bb.msg.domain.Data, "update_data()")
-    # now ask the cookie monster for help
-    #print "Cookie Monster"
-    #print "Append/Prepend %s" % d._special_values
-    #print "Overrides      %s" % d._seen_overrides
-    overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or []
-    #
-    # Well let us see what breaks here. We used to iterate
-    # over each variable and apply the override and then
-    # do the line expanding.
-    # If we have bad luck - which we will have - the keys
-    # where in some order that is so important for this
-    # method which we don't have anymore.
-    # Anyway we will fix that and write test cases this
-    # time.
-    #
-    # First we apply all overrides
-    # Then  we will handle _append and _prepend
-    #
-    for o in overrides:
-        # calculate '_'+override
-        l    = len(o)+1
-        # see if one should even try
-        if not d._seen_overrides.has_key(o):
-            continue
-        vars = d._seen_overrides[o]
-        for var in vars:
-            name = var[:-l]
-            try:
-                d[name] = d[var]
-            except:
-                bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar")
-    # now on to the appends and prepends
-    if d._special_values.has_key('_append'):
-        appends = d._special_values['_append'] or []
-        for append in appends:
-            for (a, o) in getVarFlag(append, '_append', d) or []:
-                # maybe the OVERRIDE was not yet added so keep the append
-                if (o and o in overrides) or not o:
-                    delVarFlag(append, '_append', d)
-                if o and not o in overrides:
-                    continue
-                sval = getVar(append,d) or ""
-                sval+=a
-                setVar(append, sval, d)
-    if d._special_values.has_key('_prepend'):
-        prepends = d._special_values['_prepend'] or []
-        for prepend in prepends:
-            for (a, o) in getVarFlag(prepend, '_prepend', d) or []:
-                # maybe the OVERRIDE was not yet added so keep the prepend
-                if (o and o in overrides) or not o:
-                    delVarFlag(prepend, '_prepend', d)
-                if o and not o in overrides:
-                    continue
-                sval = a + (getVar(prepend,d) or "")
-                setVar(prepend, sval, d)
 def inherits_class(klass, d):
    val = getVar('__inherit_cache', d) or []
    if os.path.join('classes', '%s.bbclass' % klass) in val:
        return True
    return False
-def _test():
-    """Start a doctest run on this module"""
-    import doctest
-    import bb
-    from bb import data
-    bb.msg.set_debug_level(0)
-    doctest.testmod(data)
-if __name__ == "__main__":
-    _test()
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 9067d54bfa..01a3330245 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -28,22 +28,16 @@ BitBake build tools.
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
-import copy, os, re, sys, time, types
+import copy, re, sys
 import bb
-from bb   import utils, methodpool
+from bb   import utils
-from COW  import COWDictBase
+from bb.COW  import COWDictBase
-from new  import classobj
-__setvar_keyword__ = ["_append","_prepend"]
+__setvar_keyword__ = ["_append", "_prepend"]
 __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?')
 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
-_expand_globals = {
-    "os": os,
-    "bb": bb,
-    "time": time,
-}
 class DataSmart:
@@ -55,9 +49,8 @@ class DataSmart:
        self._seen_overrides = seen
        self.expand_cache = {}
-        self.expand_locals = {"d": self}
-    def expand(self,s, varname):
+    def expand(self, s, varname):
        def var_sub(match):
            key = match.group()[2:-1]
            if varname and key:
@@ -70,13 +63,12 @@ class DataSmart:
                return match.group()
        def python_sub(match):
-            import bb
            code = match.group()[3:-1]
-            s = eval(code, _expand_globals, self.expand_locals)
+            codeobj = compile(code.strip(), varname or "<expansion>", "eval")
-            if type(s) == types.IntType: s = str(s)
+            value = utils.better_eval(codeobj, {"d": self})
-            return s
+            return str(value)
-        if type(s) is not types.StringType: # sanity check
+        if not isinstance(s, basestring): # sanity check
            return s
        if varname and varname in self.expand_cache:
@@ -87,9 +79,8 @@ class DataSmart:
            try:
                s = __expand_var_regexp__.sub(var_sub, s)
                s = __expand_python_regexp__.sub(python_sub, s)
-                if s == olds: break
+                if s == olds:
-                if type(s) is not types.StringType: # sanity check
+                    break
-                    bb.msg.error(bb.msg.domain.Data, 'expansion of %s returned non-string %s' % (olds, s))
            except KeyboardInterrupt:
                raise
            except:
@@ -101,23 +92,86 @@ class DataSmart:
        return s
+    def finalize(self):
+        """Performs final steps upon the datastore, including application of overrides"""
+        overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+        #
+        # Well let us see what breaks here. We used to iterate
+        # over each variable and apply the override and then
+        # do the line expanding.
+        # If we have bad luck - which we will have - the keys
+        # where in some order that is so important for this
+        # method which we don't have anymore.
+        # Anyway we will fix that and write test cases this
+        # time.
+        #
+        # First we apply all overrides
+        # Then  we will handle _append and _prepend
+        #
+        for o in overrides:
+            # calculate '_'+override
+            l = len(o) + 1
+            # see if one should even try
+            if o not in self._seen_overrides:
+                continue
+            vars = self._seen_overrides[o]
+            for var in vars:
+                name = var[:-l]
+                try:
+                    self[name] = self[var]
+                except Exception:
+                    bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar")
+        # now on to the appends and prepends
+        if "_append" in self._special_values:
+            appends = self._special_values["_append"] or []
+            for append in appends:
+                for (a, o) in self.getVarFlag(append, "_append") or []:
+                    # maybe the OVERRIDE was not yet added so keep the append
+                    if (o and o in overrides) or not o:
+                        self.delVarFlag(append, "_append")
+                    if o and not o in overrides:
+                        continue
+                    sval = self.getVar(append, False) or ""
+                    sval += a
+                    self.setVar(append, sval)
+        if "_prepend" in self._special_values:
+            prepends = self._special_values["_prepend"] or []
+            for prepend in prepends:
+                for (a, o) in self.getVarFlag(prepend, "_prepend") or []:
+                    # maybe the OVERRIDE was not yet added so keep the prepend
+                    if (o and o in overrides) or not o:
+                        self.delVarFlag(prepend, "_prepend")
+                    if o and not o in overrides:
+                        continue
+                    sval = a + (self.getVar(prepend, False) or "")
+                    self.setVar(prepend, sval)
    def initVar(self, var):
        self.expand_cache = {}
        if not var in self.dict:
            self.dict[var] = {}
-    def _findVar(self,var):
+    def _findVar(self, var):
-        _dest = self.dict
+        dest = self.dict
+        while dest:
+            if var in dest:
+                return dest[var]
-        while (_dest and var not in _dest):
+            if "_data" not in dest:
-            if not "_data" in _dest:
-                _dest = None
                break
-            _dest = _dest["_data"]
+            dest = dest["_data"]
-        if _dest and var in _dest:
-            return _dest[var]
-        return None
    def _makeShadowCopy(self, var):
        if var in self.dict:
@@ -130,7 +184,7 @@ class DataSmart:
        else:
            self.initVar(var)
-    def setVar(self,var,value):
+    def setVar(self, var, value):
        self.expand_cache = {}
        match  = __setvar_regexp__.match(var)
        if match and match.group("keyword") in __setvar_keyword__:
@@ -145,7 +199,7 @@ class DataSmart:
            # pay the cookie monster
            try:
                self._special_values[keyword].add( base )
-            except:
+            except KeyError:
                self._special_values[keyword] = set()
                self._special_values[keyword].add( base )
@@ -157,23 +211,23 @@ class DataSmart:
        # more cookies for the cookie monster
        if '_' in var:
            override = var[var.rfind('_')+1:]
-            if not self._seen_overrides.has_key(override):
+            if override not in self._seen_overrides:
                self._seen_overrides[override] = set()
            self._seen_overrides[override].add( var )
        # setting var
        self.dict[var]["content"] = value
-    def getVar(self,var,exp):
+    def getVar(self, var, exp):
-        value = self.getVarFlag(var,"content")
+        value = self.getVarFlag(var, "content")
        if exp and value:
-            return self.expand(value,var)
+            return self.expand(value, var)
        return value
    def renameVar(self, key, newkey):
        """
-        Rename the variable key to newkey 
+        Rename the variable key to newkey
        """
        val = self.getVar(key, 0)
        if val is not None:
@@ -187,30 +241,30 @@ class DataSmart:
            dest = self.getVarFlag(newkey, i) or []
            dest.extend(src)
            self.setVarFlag(newkey, i, dest)
-            
-            if self._special_values.has_key(i) and key in self._special_values[i]:
+            if i in self._special_values and key in self._special_values[i]:
                self._special_values[i].remove(key)
                self._special_values[i].add(newkey)
        self.delVar(key)
-    def delVar(self,var):
+    def delVar(self, var):
        self.expand_cache = {}
        self.dict[var] = {}
-    def setVarFlag(self,var,flag,flagvalue):
+    def setVarFlag(self, var, flag, flagvalue):
        if not var in self.dict:
            self._makeShadowCopy(var)
        self.dict[var][flag] = flagvalue
-    def getVarFlag(self,var,flag):
+    def getVarFlag(self, var, flag):
        local_var = self._findVar(var)
        if local_var:
            if flag in local_var:
                return copy.copy(local_var[flag])
        return None
-    def delVarFlag(self,var,flag):
+    def delVarFlag(self, var, flag):
        local_var = self._findVar(var)
        if not local_var:
            return
@@ -220,7 +274,7 @@ class DataSmart:
        if var in self.dict and flag in self.dict[var]:
            del self.dict[var][flag]
-    def setVarFlags(self,var,flags):
+    def setVarFlags(self, var, flags):
        if not var in self.dict:
            self._makeShadowCopy(var)
@@ -229,7 +283,7 @@ class DataSmart:
                continue
            self.dict[var][i] = flags[i]
-    def getVarFlags(self,var):
+    def getVarFlags(self, var):
        local_var = self._findVar(var)
        flags = {}
@@ -244,7 +298,7 @@ class DataSmart:
        return flags
-    def delVarFlags(self,var):
+    def delVarFlags(self, var):
        if not var in self.dict:
            self._makeShadowCopy(var)
@@ -274,21 +328,19 @@ class DataSmart:
    def keys(self):
        def _keys(d, mykey):
            if "_data" in d:
-                _keys(d["_data"],mykey)
+                _keys(d["_data"], mykey)
            for key in d.keys():
                if key != "_data":
                    mykey[key] = None
        keytab = {}
-        _keys(self.dict,keytab)
+        _keys(self.dict, keytab)
        return keytab.keys()
-    def __getitem__(self,item):
+    def __getitem__(self, item):
        #print "Warning deprecated"
        return self.getVar(item, False)
-    def __setitem__(self,var,data):
+    def __setitem__(self, var, data):
        #print "Warning deprecated"
-        self.setVar(var,data)
+        self.setVar(var, data)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index afd5bf57c1..7731649eff 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -22,7 +22,8 @@ BitBake build tools.
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import os, re, sys
+import os, sys
+import warnings
 import bb.utils
 import pickle
@@ -38,7 +39,7 @@ class Event:
        self.pid = worker_pid
 NotHandled = 0
-Handled = 1
+Handled    = 1
 Registered        = 10
 AlreadyRegistered = 14
@@ -48,13 +49,25 @@ _handlers = {}
 _ui_handlers = {}
 _ui_handler_seq = 0
+# For compatibility
+bb.utils._context["NotHandled"] = NotHandled
+bb.utils._context["Handled"] = Handled
 def fire_class_handlers(event, d):
+    import bb.msg
+    if isinstance(event, bb.msg.MsgBase):
+        return
    for handler in _handlers:
        h = _handlers[handler]
        event.data = d
        if type(h).__name__ == "code":
-            exec(h)
+            locals = {"e": event}
-            tmpHandler(event)
+            bb.utils.simple_exec(h, locals)
+            ret = bb.utils.better_eval("tmpHandler(e)", locals)
+            if ret is not None:
+                warnings.warn("Using Handled/NotHandled in event handlers is deprecated",
+                              DeprecationWarning, stacklevel = 2)
        else:
            h(event)
        del event.data
@@ -76,9 +89,9 @@ def fire_ui_handlers(event, d):
 def fire(event, d):
    """Fire off an Event"""
-    # We can fire class handlers in the worker process context and this is 
+    # We can fire class handlers in the worker process context and this is
    # desired so they get the task based datastore.
-    # UI handlers need to be fired in the server context so we defer this. They 
+    # UI handlers need to be fired in the server context so we defer this. They
    # don't have a datastore so the datastore context isn't a problem.
    fire_class_handlers(event, d)
@@ -91,13 +104,13 @@ def worker_fire(event, d):
    data = "<event>" + pickle.dumps(event) + "</event>"
    try:
        if os.write(worker_pipe, data) != len (data):
-            print "Error sending event to server (short write)"
+            print("Error sending event to server (short write)")
    except OSError:
        sys.exit(1)
 def fire_from_worker(event, d):
    if not event.startswith("<event>") or not event.endswith("</event>"):
-        print "Error, not an event"
+        print("Error, not an event")
        return
    event = pickle.loads(event[7:-8])
    fire_ui_handlers(event, d)
@@ -222,10 +235,11 @@ class BuildCompleted(BuildBase):
 class NoProvider(Event):
    """No Provider for an Event"""
-    def __init__(self, item, runtime=False):
+    def __init__(self, item, runtime=False, dependees=None):
        Event.__init__(self)
        self._item = item
        self._runtime = runtime
+        self._dependees = dependees
    def getItem(self):
        return self._item
@@ -284,4 +298,3 @@ class DepTreeGenerated(Event):
    def __init__(self, depgraph):
        Event.__init__(self)
        self._depgraph = depgraph
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index b566da4311..31b9653793 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -24,6 +24,8 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+from __future__ import absolute_import
+from __future__ import print_function
 import os, re
 import bb
 from   bb import data
@@ -53,24 +55,6 @@ class InvalidSRCREV(Exception):
 def decodeurl(url):
    """Decodes an URL into the tokens (scheme, network location, path,
    user, password, parameters).
-    >>> decodeurl("http://www.google.com/index.html")
-    ('http', 'www.google.com', '/index.html', '', '', {})
-    >>> decodeurl("file://gas/COPYING")
-    ('file', '', 'gas/COPYING', '', '', {})
-    CVS url with username, host and cvsroot. The cvs module to check out is in the
-    parameters:
-    >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg")
-    ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'})
-    Dito, but this time the username has a password part. And we also request a special tag
-    to check out.
-    >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
-    ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
    """
    m = re.compile('(?P<type>[^:]*)://((?P<user>.+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
@@ -103,7 +87,7 @@ def decodeurl(url):
    p = {}
    if parm:
        for s in parm.split(';'):
-            s1,s2 = s.split('=')
+            s1, s2 = s.split('=')
            p[s1] = s2
    return (type, host, path, user, pswd, p)
@@ -111,27 +95,12 @@ def decodeurl(url):
 def encodeurl(decoded):
    """Encodes a URL from tokens (scheme, network location, path,
    user, password, parameters).
-    >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}])
-    'http://www.google.com/index.html'
-    CVS with username, host and cvsroot. The cvs module to check out is in the
-    parameters:
-    >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}])
-    'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg'
-    Dito, but this time the username has a password part. And we also request a special tag
-    to check out.
-    >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}])
-    'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg'
    """
    (type, host, path, user, pswd, p) = decoded
    if not type or not path:
-        bb.msg.fatal(bb.msg.domain.Fetcher, "invalid or missing parameters for url encoding")
+        raise MissingParameterError("Type or path url components missing when encoding %s" % decoded)
    url = '%s://' % type
    if user:
        url += "%s" % user
@@ -151,15 +120,14 @@ def uri_replace(uri, uri_find, uri_replace, d):
 #   bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
    if not uri or not uri_find or not uri_replace:
        bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
-    uri_decoded = list(bb.decodeurl(uri))
+    uri_decoded = list(decodeurl(uri))
-    uri_find_decoded = list(bb.decodeurl(uri_find))
+    uri_find_decoded = list(decodeurl(uri_find))
-    uri_replace_decoded = list(bb.decodeurl(uri_replace))
+    uri_replace_decoded = list(decodeurl(uri_replace))
-    result_decoded = ['','','','','',{}]
+    result_decoded = ['', '', '', '', '', {}]
    for i in uri_find_decoded:
        loc = uri_find_decoded.index(i)
        result_decoded[loc] = uri_decoded[loc]
-        import types
+        if isinstance(i, basestring):
-        if type(i) == types.StringType:
            if (re.match(i, uri_decoded[loc])):
                result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
                if uri_find_decoded.index(i) == 2:
@@ -174,19 +142,20 @@ def uri_replace(uri, uri_find, uri_replace, d):
 #           else:
 #               for j in i:
 #                   FIXME: apply replacements against options
-    return bb.encodeurl(result_decoded)
+    return encodeurl(result_decoded)
 methods = []
 urldata_cache = {}
 saved_headrevs = {}
+persistent_database_connection = {}
 def fetcher_init(d):
    """
-    Called to initilize the fetchers once the configuration data is known
+    Called to initialize the fetchers once the configuration data is known.
    Calls before this must not hit the cache.
    """
-    pd = persist_data.PersistData(d)
+    pd = persist_data.PersistData(d, persistent_database_connection)
-    # When to drop SCM head revisions controled by user policy
+    # When to drop SCM head revisions controlled by user policy
    srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear"
    if srcrev_policy == "cache":
        bb.msg.debug(1, bb.msg.domain.Fetcher, "Keeping SRCREV cache due to cache policy of: %s" % srcrev_policy)
@@ -198,7 +167,7 @@ def fetcher_init(d):
            pass
        pd.delDomain("BB_URI_HEADREVS")
    else:
-        bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy)
+        raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
    for m in methods:
        if hasattr(m, "init"):
@@ -214,7 +183,7 @@ def fetcher_compare_revisons(d):
    return true/false on whether they've changed.
    """
-    pd = persist_data.PersistData(d)
+    pd = persist_data.PersistData(d, persistent_database_connection)
    data = pd.getKeyValues("BB_URI_HEADREVS")
    data2 = bb.fetch.saved_headrevs
@@ -236,6 +205,7 @@ def fetcher_compare_revisons(d):
 def init(urls, d, setup = True):
    urldata = {}
    fn = bb.data.getVar('FILE', d, 1)
    if fn in urldata_cache:
        urldata = urldata_cache[fn]
@@ -247,7 +217,7 @@ def init(urls, d, setup = True):
    if setup:
        for url in urldata:
            if not urldata[url].setup:
-                urldata[url].setup_localpath(d) 
+                urldata[url].setup_localpath(d)
    urldata_cache[fn] = urldata
    return urldata
@@ -265,7 +235,7 @@ def go(d, urls = None):
        ud = urldata[u]
        m = ud.method
        if ud.localfile:
-            if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
+            if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5) and os.path.exists(ud.localfile):
                # File already present along with md5 stamp file
                # Touch md5 file to show activity
                try:
@@ -275,8 +245,8 @@ def go(d, urls = None):
                    pass
                continue
            lf = bb.utils.lockfile(ud.lockfile)
-            if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
+            if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5) and os.path.exists(ud.localfile):
-                # If someone else fetched this before we got the lock, 
+                # If someone else fetched this before we got the lock,
                # notice and don't try again
                try:
                    os.utime(ud.md5, None)
@@ -332,7 +302,7 @@ def checkstatus(d):
                ret = try_mirrors (d, u, mirrors, True)
        if not ret:
-            bb.msg.error(bb.msg.domain.Fetcher, "URL %s doesn't work" % u)
+            raise FetchError("URL %s doesn't work" % u)
 def localpaths(d):
    """
@@ -342,7 +312,7 @@ def localpaths(d):
    urldata = init([], d, True)
    for u in urldata:
-        ud = urldata[u]      
+        ud = urldata[u]
        local.append(ud.localpath)
    return local
@@ -354,15 +324,15 @@ def get_srcrev(d):
    Return the version string for the current package
    (usually to be used as PV)
    Most packages usually only have one SCM so we just pass on the call.
-    In the multi SCM case, we build a value based on SRCREV_FORMAT which must 
+    In the multi SCM case, we build a value based on SRCREV_FORMAT which must
    have been set.
    """
    #
-    # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which 
+    # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which
    # could translate into a call to here. If it does, we need to catch this
    # and provide some way so it knows get_srcrev is active instead of being
-    # some number etc. hence the srcrev_internal_call tracking and the magic  
+    # some number etc. hence the srcrev_internal_call tracking and the magic
    # "SRCREVINACTION" return value.
    #
    # Neater solutions welcome!
@@ -372,7 +342,7 @@ def get_srcrev(d):
    scms = []
-    # Only call setup_localpath on URIs which suppports_srcrev() 
+    # Only call setup_localpath on URIs which suppports_srcrev()
    urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d, False)
    for u in urldata:
        ud = urldata[u]
@@ -385,7 +355,7 @@ def get_srcrev(d):
        bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI")
        raise ParameterError
-    bb.data.setVar('__BB_DONT_CACHE','1', d)
+    bb.data.setVar('__BB_DONT_CACHE', '1', d)
    if len(scms) == 1:
        return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d)
@@ -408,7 +378,7 @@ def get_srcrev(d):
 def localpath(url, d, cache = True):
    """
-    Called from the parser with cache=False since the cache isn't ready 
+    Called from the parser with cache=False since the cache isn't ready
    at this point. Also called from classed in OE e.g. patch.bbclass
    """
    ud = init([url], d)
@@ -432,7 +402,7 @@ def runfetchcmd(cmd, d, quiet = False):
    for var in exportvars:
        val = data.getVar(var, d, True)
        if val:
-            cmd = 'export ' + var + '=%s; %s' % (val, cmd)
+            cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
    bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd)
@@ -440,12 +410,12 @@ def runfetchcmd(cmd, d, quiet = False):
    stdout_handle = os.popen(cmd + " 2>&1", "r")
    output = ""
-    while 1:
+    while True:
        line = stdout_handle.readline()
        if not line:
            break
        if not quiet:
-            print line,
+            print(line, end=' ')
        output += line
    status =  stdout_handle.close() or 0
@@ -507,7 +477,7 @@ class FetchData(object):
    """
    def __init__(self, url, d):
        self.localfile = ""
-        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d))
+        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d))
        self.date = Fetch.getSRCDate(self, d)
        self.url = url
        if not self.user and "user" in self.parm:
@@ -571,7 +541,7 @@ class Fetch(object):
    def localpath(self, url, urldata, d):
        """
        Return the local filename of a given url assuming a successful fetch.
-        Can also setup variables in urldata for use in go (saving code duplication 
+        Can also setup variables in urldata for use in go (saving code duplication
        and duplicate code execution)
        """
        return url
@@ -632,8 +602,8 @@ class Fetch(object):
        """
        Return:
            a) a source revision if specified
-            b) True if auto srcrev is in action
+            b) True if auto srcrev is in action
-            c) False otherwise
+            c) False otherwise
        """
        if 'rev' in ud.parm:
@@ -665,7 +635,7 @@ class Fetch(object):
            b) None otherwise
        """
-        localcount= None
+        localcount = None
        if 'name' in ud.parm:
            pn = data.getVar("PN", d, 1)
            localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1)
@@ -706,7 +676,7 @@ class Fetch(object):
        if not hasattr(self, "_latest_revision"):
            raise ParameterError
-        pd = persist_data.PersistData(d)
+        pd = persist_data.PersistData(d, persistent_database_connection)
        key = self.generate_revision_key(url, ud, d)
        rev = pd.getValue("BB_URI_HEADREVS", key)
        if rev != None:
@@ -718,12 +688,12 @@ class Fetch(object):
    def sortable_revision(self, url, ud, d):
        """
-        
        """
        if hasattr(self, "_sortable_revision"):
            return self._sortable_revision(url, ud, d)
-        pd = persist_data.PersistData(d)
+        pd = persist_data.PersistData(d, persistent_database_connection)
        key = self.generate_revision_key(url, ud, d)
        latest_rev = self._build_revision(url, ud, d)
@@ -758,18 +728,18 @@ class Fetch(object):
        key = self._revision_key(url, ud, d)
        return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
-import cvs
+from . import cvs
-import git
+from . import git
-import local
+from . import local
-import svn
+from . import svn
-import wget
+from . import wget
-import svk
+from . import svk
-import ssh
+from . import ssh
-import perforce
+from . import perforce
-import bzr
+from . import bzr
-import hg
+from . import hg
-import osc
+from . import osc
-import repo
+from . import repo
 methods.append(local.Local())
 methods.append(wget.Wget())
diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py
index c6e33c3343..813d7d8c80 100644
--- a/bitbake/lib/bb/fetch/bzr.py
+++ b/bitbake/lib/bb/fetch/bzr.py
@@ -46,15 +46,15 @@ class Bzr(Fetch):
        revision = Fetch.srcrev_internal_helper(ud, d)
        if revision is True:
-            ud.revision = self.latest_revision(url, ud, d)      
+            ud.revision = self.latest_revision(url, ud, d)
        elif revision:
            ud.revision = revision
        if not ud.revision:
-            ud.revision = self.latest_revision(url, ud, d)      
+            ud.revision = self.latest_revision(url, ud, d)
        ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
-        
        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
    def _buildbzrcommand(self, ud, d, command):
@@ -145,4 +145,3 @@ class Bzr(Fetch):
    def _build_revision(self, url, ud, d):
        return ud.revision
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 443f521317..61976f7ef4 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -139,8 +139,8 @@ class Cvs(Fetch):
        bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory")
        pkg = data.expand('${PN}', d)
        pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
-        moddir = os.path.join(pkgdir,localdir)
+        moddir = os.path.join(pkgdir, localdir)
-        if os.access(os.path.join(moddir,'CVS'), os.R_OK):
+        if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
            bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
            # update sources there
            os.chdir(moddir)
@@ -157,7 +157,7 @@ class Cvs(Fetch):
            try:
                os.rmdir(moddir)
            except OSError:
-                 pass
+                pass
            raise FetchError(ud.module)
        # tar them up to a defined filename
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 41ebc5b998..8c91de9db1 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -57,12 +57,12 @@ class Git(Fetch):
        tag = Fetch.srcrev_internal_helper(ud, d)
        if tag is True:
-            ud.tag = self.latest_revision(url, ud, d)   
+            ud.tag = self.latest_revision(url, ud, d)
        elif tag:
            ud.tag = tag
        if not ud.tag or ud.tag == "master":
-            ud.tag = self.latest_revision(url, ud, d)   
+            ud.tag = self.latest_revision(url, ud, d)
        subdir = ud.parm.get("subpath", "")
        if subdir != "":
@@ -114,7 +114,7 @@ class Git(Fetch):
        os.chdir(ud.clonedir)
        mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
-        if mirror_tarballs != "0" or 'fullclone' in ud.parm: 
+        if mirror_tarballs != "0" or 'fullclone' in ud.parm:
            bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
            runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
@@ -188,7 +188,7 @@ class Git(Fetch):
    def _sortable_buildindex_disabled(self, url, ud, d, rev):
        """
-        Return a suitable buildindex for the revision specified. This is done by counting revisions 
+        Return a suitable buildindex for the revision specified. This is done by counting revisions
        using "git rev-list" which may or may not work in different circumstances.
        """
@@ -197,7 +197,7 @@ class Git(Fetch):
        # Check if we have the rev already
        if not os.path.exists(ud.clonedir):
-            print "no repo"
+            print("no repo")
            self.go(None, ud, d)
            if not os.path.exists(ud.clonedir):
                bb.msg.error(bb.msg.domain.Fetcher, "GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value" % (url, ud.clonedir))
@@ -213,5 +213,4 @@ class Git(Fetch):
        buildindex = "%s" % output.split()[0]
        bb.msg.debug(1, bb.msg.domain.Fetcher, "GIT repository for %s in %s is returning %s revisions in rev-list before %s" % (url, ud.clonedir, buildindex, rev))
-        return buildindex        
+        return buildindex
diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py
index d0756382f8..efb3b5c76d 100644
--- a/bitbake/lib/bb/fetch/hg.py
+++ b/bitbake/lib/bb/fetch/hg.py
@@ -134,9 +134,9 @@ class Hg(Fetch):
            os.chdir(ud.pkgdir)
            bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd)
            runfetchcmd(fetchcmd, d)
-        
-        # Even when we clone (fetch), we still need to update as hg's clone
+        # Even when we clone (fetch), we still need to update as hg's clone
-        # won't checkout the specified revision if its on a branch
+        # won't checkout the specified revision if its on a branch
        updatecmd = self._buildhgcommand(ud, d, "update")
        bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
        runfetchcmd(updatecmd, d)
@@ -170,4 +170,3 @@ class Hg(Fetch):
        Return a unique key for the url
        """
        return "hg:" + ud.moddir
diff --git a/bitbake/lib/bb/fetch/local.py b/bitbake/lib/bb/fetch/local.py
index f9bdf589cb..882a2c4602 100644
--- a/bitbake/lib/bb/fetch/local.py
+++ b/bitbake/lib/bb/fetch/local.py
@@ -27,6 +27,7 @@ BitBake build tools.
 import os
 import bb
+import bb.utils
 from   bb import data
 from   bb.fetch import Fetch
@@ -47,7 +48,7 @@ class Local(Fetch):
        if path[0] != "/":
            filespath = data.getVar('FILESPATH', d, 1)
            if filespath:
-                newpath = bb.which(filespath, path)
+                newpath = bb.utils.which(filespath, path)
            if not newpath:
                filesdir = data.getVar('FILESDIR', d, 1)
                if filesdir:
@@ -65,8 +66,8 @@ class Local(Fetch):
        Check the status of the url
        """
        if urldata.localpath.find("*") != -1:
-            bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s looks like a glob and was therefore not checked." % url)      
+            bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s looks like a glob and was therefore not checked." % url)
-            return True
+            return True
        if os.path.exists(urldata.localpath):
-           return True
+            return True
        return False
diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py
index 548dd9d074..ed773939b0 100644
--- a/bitbake/lib/bb/fetch/osc.py
+++ b/bitbake/lib/bb/fetch/osc.py
@@ -16,7 +16,7 @@ from    bb.fetch import MissingParameterError
 from    bb.fetch import runfetchcmd
 class Osc(Fetch):
-    """Class to fetch a module or modules from Opensuse build server 
+    """Class to fetch a module or modules from Opensuse build server
       repositories."""
    def supports(self, url, ud, d):
@@ -64,7 +64,7 @@ class Osc(Fetch):
        proto = "ocs"
        if "proto" in ud.parm:
            proto = ud.parm["proto"]
-        
        options = []
        config = "-c %s" % self.generate_config(ud, d)
@@ -108,7 +108,7 @@ class Osc(Fetch):
            os.chdir(ud.pkgdir)
            bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd)
            runfetchcmd(oscfetchcmd, d)
-        
        os.chdir(os.path.join(ud.pkgdir + ud.path))
        # tar them up to a defined filename
        try:
@@ -131,7 +131,7 @@ class Osc(Fetch):
        config_path = "%s/oscrc" % data.expand('${OSCDIR}', d)
        if (os.path.exists(config_path)):
-            os.remove(config_path) 
+            os.remove(config_path)
        f = open(config_path, 'w')
        f.write("[general]\n")
@@ -146,5 +146,5 @@ class Osc(Fetch):
        f.write("user = %s\n" % ud.parm["user"])
        f.write("pass = %s\n" % ud.parm["pswd"])
        f.close()
-        
        return config_path
diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py
index e2c3421089..1c74cff349 100644
--- a/bitbake/lib/bb/fetch/perforce.py
+++ b/bitbake/lib/bb/fetch/perforce.py
@@ -25,6 +25,7 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
+from future_builtins import zip
 import os
 import bb
 from   bb import data
@@ -35,15 +36,15 @@ class Perforce(Fetch):
    def supports(self, url, ud, d):
        return ud.type in ['p4']
-    def doparse(url,d):
+    def doparse(url, d):
        parm = {}
        path = url.split("://")[1]
        delim = path.find("@");
        if delim != -1:
-            (user,pswd,host,port) = path.split('@')[0].split(":")
+            (user, pswd, host, port) = path.split('@')[0].split(":")
            path = path.split('@')[1]
        else:
-            (host,port) = data.getVar('P4PORT', d).split(':')
+            (host, port) = data.getVar('P4PORT', d).split(':')
            user = ""
            pswd = ""
@@ -53,19 +54,19 @@ class Perforce(Fetch):
            plist = path.split(';')
            for item in plist:
                if item.count('='):
-                    (key,value) = item.split('=')
+                    (key, value) = item.split('=')
                    keys.append(key)
                    values.append(value)
-            parm = dict(zip(keys,values))
+            parm = dict(zip(keys, values))
        path = "//" + path.split(';')[0]
        host += ":%s" % (port)
        parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
-        return host,path,user,pswd,parm
+        return host, path, user, pswd, parm
    doparse = staticmethod(doparse)
-    def getcset(d, depot,host,user,pswd,parm):
+    def getcset(d, depot, host, user, pswd, parm):
        p4opt = ""
        if "cset" in parm:
            return parm["cset"];
@@ -95,9 +96,9 @@ class Perforce(Fetch):
        return cset.split(' ')[1]
    getcset = staticmethod(getcset)
-    def localpath(self, url, ud, d):        
+    def localpath(self, url, ud, d):
-        (host,path,user,pswd,parm) = Perforce.doparse(url,d)
+        (host, path, user, pswd, parm) = Perforce.doparse(url, d)
        # If a label is specified, we use that as our filename
@@ -115,7 +116,7 @@ class Perforce(Fetch):
        cset = Perforce.getcset(d, path, host, user, pswd, parm)
-        ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host,base.replace('/', '.'), cset), d)
+        ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d)
        return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
@@ -124,7 +125,7 @@ class Perforce(Fetch):
        Fetch urls
        """
-        (host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
+        (host, depot, user, pswd, parm) = Perforce.doparse(loc, d)
        if depot.find('/...') != -1:
            path = depot[:depot.find('/...')]
@@ -160,14 +161,14 @@ class Perforce(Fetch):
        tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
        tmpfile = tmppipe.readline().strip()
        if not tmpfile:
-            bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+            bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
            raise FetchError(module)
        if "label" in parm:
-            depot = "%s@%s" % (depot,parm["label"])
+            depot = "%s@%s" % (depot, parm["label"])
        else:
            cset = Perforce.getcset(d, depot, host, user, pswd, parm)
-            depot = "%s@%s" % (depot,cset)
+            depot = "%s@%s" % (depot, cset)
        os.chdir(tmpfile)
        bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
@@ -175,12 +176,12 @@ class Perforce(Fetch):
        p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot))
        if not p4file:
-            bb.error("Fetch: unable to get the P4 files from %s" % (depot))
+            bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to get the P4 files from %s" % (depot))
            raise FetchError(module)
        count = 0
-        for file in p4file: 
+        for file in p4file:
            list = file.split()
            if list[2] == "delete":
@@ -189,11 +190,11 @@ class Perforce(Fetch):
            dest = list[0][len(path)+1:]
            where = dest.find("#")
-            os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module,dest[:where],list[0]))
+            os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0]))
            count = count + 1
-            
        if count == 0:
-            bb.error("Fetch:  No files gathered from the P4 fetch")
+            bb.msg.error(bb.msg.domain.Fetcher, "Fetch:  No files gathered from the P4 fetch")
            raise FetchError(module)
        myret = os.system("tar -czf %s %s" % (ud.localpath, module))
@@ -205,5 +206,3 @@ class Perforce(Fetch):
            raise FetchError(module)
        # cleanup
        os.system('rm -rf %s' % tmpfile)
diff --git a/bitbake/lib/bb/fetch/repo.py b/bitbake/lib/bb/fetch/repo.py
index 34c32fe0bb..883310b019 100644
--- a/bitbake/lib/bb/fetch/repo.py
+++ b/bitbake/lib/bb/fetch/repo.py
@@ -23,11 +23,10 @@ BitBake "Fetch" repo (git) implementation
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import os, re
+import os
 import bb
 from   bb    import data
 from   bb.fetch import Fetch
-from   bb.fetch import FetchError
 from   bb.fetch import runfetchcmd
 class Repo(Fetch):
diff --git a/bitbake/lib/bb/fetch/ssh.py b/bitbake/lib/bb/fetch/ssh.py
index 68e6fdb1df..86c76f4e44 100644
--- a/bitbake/lib/bb/fetch/ssh.py
+++ b/bitbake/lib/bb/fetch/ssh.py
@@ -114,5 +114,5 @@ class SSH(Fetch):
        (exitstatus, output) = commands.getstatusoutput(cmd)
        if exitstatus != 0:
-            print output
+            print(output)
            raise FetchError('Unable to fetch %s' % url)
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index ba9f6ab109..375e8df055 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -78,7 +78,7 @@ class Svn(Fetch):
                    ud.revision = rev
                    ud.date = ""
                else:
-                    ud.revision = ""            
+                    ud.revision = ""
        ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py
index ae1c6ad136..dcc58c75e8 100644
--- a/bitbake/lib/bb/fetch/wget.py
+++ b/bitbake/lib/bb/fetch/wget.py
@@ -30,6 +30,8 @@ import bb
 from   bb import data
 from   bb.fetch import Fetch
 from   bb.fetch import FetchError
+from   bb.fetch import encodeurl, decodeurl
+from   bb.fetch import runfetchcmd
 class Wget(Fetch):
    """Class to fetch urls via 'wget'"""
@@ -37,11 +39,11 @@ class Wget(Fetch):
        """
        Check to see if a given url can be fetched with wget.
        """
-        return ud.type in ['http','https','ftp']
+        return ud.type in ['http', 'https', 'ftp']
    def localpath(self, url, ud, d):
-        url = bb.encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}])
+        url = encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}])
        ud.basename = os.path.basename(ud.path)
        ud.localfile = data.expand(os.path.basename(url), d)
@@ -60,37 +62,16 @@ class Wget(Fetch):
                fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
            uri = uri.split(";")[0]
-            uri_decoded = list(bb.decodeurl(uri))
+            uri_decoded = list(decodeurl(uri))
            uri_type = uri_decoded[0]
            uri_host = uri_decoded[1]
-            bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
            fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
            fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
-            httpproxy = None
-            ftpproxy = None
+            bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
-            if uri_type == 'http':
-                httpproxy = data.getVar("HTTP_PROXY", d, True)
-                httpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split()
-                for p in httpproxy_ignore:
-                    if uri_host.endswith(p):
-                        httpproxy = None
-                        break
-            if uri_type == 'ftp':
-                ftpproxy = data.getVar("FTP_PROXY", d, True)
-                ftpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split()
-                for p in ftpproxy_ignore:
-                    if uri_host.endswith(p):
-                        ftpproxy = None
-                        break
-            if httpproxy:
-                fetchcmd = "http_proxy=" + httpproxy + " " + fetchcmd
-            if ftpproxy:
-                fetchcmd = "ftp_proxy=" + ftpproxy + " " + fetchcmd
            bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd)
-            ret = os.system(fetchcmd)
+            runfetchcmd(fetchcmd, d)
-            if ret != 0:
-                return False
            # Sanity check since wget can pretend it succeed when it didn't
            # Also, this used to happen if sourceforge sent us to the mirror page
diff --git a/bitbake/lib/bb/methodpool.py b/bitbake/lib/bb/methodpool.py
index f43c4a0580..1485b1357d 100644
--- a/bitbake/lib/bb/methodpool.py
+++ b/bitbake/lib/bb/methodpool.py
@@ -27,7 +27,7 @@
    a method pool to do this task.
    This pool will be used to compile and execute the functions. It
-    will be smart enough to 
+    will be smart enough to
 """
 from bb.utils import better_compile, better_exec
@@ -43,8 +43,8 @@ def insert_method(modulename, code, fn):
    Add code of a module should be added. The methods
    will be simply added, no checking will be done
    """
-    comp = better_compile(code, "<bb>", fn )
+    comp = better_compile(code, modulename, fn )
-    better_exec(comp, __builtins__, code, fn)
+    better_exec(comp, None, code, fn)
    # now some instrumentation
    code = comp.co_names
@@ -59,7 +59,7 @@ def insert_method(modulename, code, fn):
 def check_insert_method(modulename, code, fn):
    """
    Add the code if it wasnt added before. The module
-    name will be used for that 
+    name will be used for that
    Variables:
        @modulename a short name e.g. base.bbclass
@@ -81,4 +81,4 @@ def get_parsed_dict():
    """
    shortcut
    """
-    return _parsed_methods    
+    return _parsed_methods
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index 3fcf7091be..cea5efb5a4 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -22,26 +22,32 @@ Message handling infrastructure for bitbake
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import sys, bb
+import sys
-from bb import event
+import collections
+import bb
-debug_level = {}
+import bb.event
+debug_level = collections.defaultdict(lambda: 0)
 verbose = False
-domain = bb.utils.Enum(
+def _NamedTuple(name, fields):
-    'Build',
+    Tuple = collections.namedtuple(name, " ".join(fields))
-    'Cache',
+    return Tuple(*range(len(fields)))
-    'Collection',
-    'Data',
+domain = _NamedTuple("Domain", (
-    'Depends',
+    "Default",
-    'Fetcher',
+    "Build",
-    'Parsing',
+    "Cache",
-    'PersistData',
+    "Collection",
-    'Provider',
+    "Data",
-    'RunQueue',
+    "Depends",
-    'TaskData',
+    "Fetcher",
-    'Util')
+    "Parsing",
+    "PersistData",
+    "Provider",
+    "RunQueue",
+    "TaskData",
+    "Util"))
 class MsgBase(bb.event.Event):
@@ -49,7 +55,7 @@ class MsgBase(bb.event.Event):
    def __init__(self, msg):
        self._message = msg
-        event.Event.__init__(self)
+        bb.event.Event.__init__(self)
 class MsgDebug(MsgBase):
    """Debug Message"""
@@ -74,52 +80,62 @@ class MsgPlain(MsgBase):
 #
 def set_debug_level(level):
-    bb.msg.debug_level = {}
+    for d in domain:
-    for domain in bb.msg.domain:
+        debug_level[d] = level
-        bb.msg.debug_level[domain] = level
+    debug_level[domain.Default] = level
-    bb.msg.debug_level['default'] = level
+def get_debug_level(msgdomain = domain.Default):
+    return debug_level[msgdomain]
 def set_verbose(level):
-    bb.msg.verbose = level
+    verbose = level
-def set_debug_domains(domains):
+def set_debug_domains(strdomains):
-    for domain in domains:
+    for domainstr in strdomains:
-        found = False
+        for d in domain:
-        for ddomain in bb.msg.domain:
+            if domain._fields[d] == domainstr:
-            if domain == str(ddomain):
+                debug_level[d] += 1
-                bb.msg.debug_level[ddomain] = bb.msg.debug_level[ddomain] + 1
+                break
-                found = True
+        else:
-        if not found:
+            warn(None, "Logging domain %s is not valid, ignoring" % domainstr)
-            bb.msg.warn(None, "Logging domain %s is not valid, ignoring" % domain)
 #
 # Message handling functions
 #
-def debug(level, domain, msg, fn = None):
+def debug(level, msgdomain, msg, fn = None):
-    if not domain:
+    if not msgdomain:
-        domain = 'default'
+        msgdomain = domain.Default
-    if debug_level[domain] >= level:
+    if debug_level[msgdomain] >= level:
        bb.event.fire(MsgDebug(msg), None)
+        if not bb.event._ui_handlers:
+            print('DEBUG: ' + msg)
+def note(level, msgdomain, msg, fn = None):
+    if not msgdomain:
+        msgdomain = domain.Default
-def note(level, domain, msg, fn = None):
+    if level == 1 or verbose or debug_level[msgdomain] >= 1:
-    if not domain:
-        domain = 'default'
-    if level == 1 or verbose or debug_level[domain] >= 1:
        bb.event.fire(MsgNote(msg), None)
+        if not bb.event._ui_handlers:
+            print('NOTE: ' + msg)
-def warn(domain, msg, fn = None):
+def warn(msgdomain, msg, fn = None):
    bb.event.fire(MsgWarn(msg), None)
+    if not bb.event._ui_handlers:
+        print('WARNING: ' + msg)
-def error(domain, msg, fn = None):
+def error(msgdomain, msg, fn = None):
    bb.event.fire(MsgError(msg), None)
    print 'ERROR: ' + msg
-def fatal(domain, msg, fn = None):
+def fatal(msgdomain, msg, fn = None):
    bb.event.fire(MsgFatal(msg), None)
-    print 'FATAL: ' + msg
+    print('FATAL: ' + msg)
    sys.exit(1)
 def plain(msg, fn = None):
    bb.event.fire(MsgPlain(msg), None)
+    if not bb.event._ui_handlers:
+        print(msg)
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index 2a7897cdf2..95f372b00b 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -24,11 +24,10 @@ File parsers for the BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
-__all__ = [ 'ParseError', 'SkipPackage', 'cached_mtime', 'mark_dependency',
-            'supports', 'handle', 'init' ]
 handlers = []
 import bb, os
+import bb.utils
 class ParseError(Exception):
    """Exception raised when parsing fails"""
@@ -38,12 +37,12 @@ class SkipPackage(Exception):
 __mtime_cache = {}
 def cached_mtime(f):
-    if not __mtime_cache.has_key(f):
+    if f not in __mtime_cache:
        __mtime_cache[f] = os.stat(f)[8]
    return __mtime_cache[f]
 def cached_mtime_noerror(f):
-    if not __mtime_cache.has_key(f):
+    if f not in __mtime_cache:
        try:
            __mtime_cache[f] = os.stat(f)[8]
        except OSError:
@@ -57,8 +56,8 @@ def update_mtime(f):
 def mark_dependency(d, f):
    if f.startswith('./'):
        f = "%s/%s" % (os.getcwd(), f[2:])
-    deps = bb.data.getVar('__depends', d) or []
+    deps = bb.data.getVar('__depends', d) or set()
-    deps.append( (f, cached_mtime(f)) )
+    deps.update([(f, cached_mtime(f))])
    bb.data.setVar('__depends', deps, d)
 def supports(fn, data):
@@ -82,9 +81,11 @@ def init(fn, data):
 def resolve_file(fn, d):
    if not os.path.isabs(fn):
-        fn = bb.which(bb.data.getVar("BBPATH", d, 1), fn)
+        bbpath = bb.data.getVar("BBPATH", d, True)
-        if not fn:
+        newfn = bb.which(bbpath, fn)
-            raise IOError("file %s not found" % fn)
+        if not newfn:
+            raise IOError("file %s not found in %s" % (fn, bbpath))
+        fn = newfn
    bb.msg.debug(2, bb.msg.domain.Parsing, "LOAD %s" % fn)
    return fn
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index 59aa44bee0..dae2e11154 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -21,8 +21,11 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+from __future__ import absolute_import
+from future_builtins import filter
 import bb, re, string
-from itertools import chain
+from bb import methodpool
+import itertools
 __word__ = re.compile(r"\S+")
 __parsed_methods__ = bb.methodpool.get_parsed_dict()
@@ -30,7 +33,8 @@ _bbversions_re = re.compile(r"\[(?P<from>[0-9]+)-(?P<to>[0-9]+)\]")
 class StatementGroup(list):
    def eval(self, data):
-        map(lambda x: x.eval(data), self)
+        for statement in self:
+            statement.eval(data)
 class AstNode(object):
    pass
@@ -103,7 +107,6 @@ class DataNode(AstNode):
            val = groupd["value"]
        if 'flag' in groupd and groupd['flag'] != None:
-            bb.msg.debug(3, bb.msg.domain.Parsing, "setVarFlag(%s, %s, %s, data)" % (key, groupd['flag'], val))
            bb.data.setVarFlag(key, groupd['flag'], val, data)
        elif groupd["lazyques"]:
            assigned = bb.data.getVar("__lazy_assigned", data) or []
@@ -143,7 +146,7 @@ class PythonMethodNode(AstNode):
        # Note we will add root to parsedmethods after having parse
        # 'this' file. This means we will not parse methods from
        # bb classes twice
-        if not self.root  in __parsed_methods__:
+        if not bb.methodpool.parsed_module(self.root):
            text = '\n'.join(self.body)
            bb.methodpool.insert_method(self.root, text, self.fn)
@@ -254,7 +257,7 @@ class InheritNode(AstNode):
    def eval(self, data):
        bb.parse.BBHandler.inherit(self.n, data)
- 
 def handleInclude(statements, m, fn, lineno, force):
    statements.append(IncludeNode(m.group(1), fn, lineno, force))
@@ -293,7 +296,7 @@ def handleInherit(statements, m):
    n = __word__.findall(files)
    statements.append(InheritNode(m.group(1)))
-def finalise(fn, d):
+def finalize(fn, d):
    for lazykey in bb.data.getVar("__lazy_assigned", d) or ():
        if bb.data.getVar(lazykey, d) is None:
            val = bb.data.getVarFlag(lazykey, "defaultval", d)
@@ -301,35 +304,16 @@ def finalise(fn, d):
    bb.data.expandKeys(d)
    bb.data.update_data(d)
-    anonqueue = bb.data.getVar("__anonqueue", d, 1) or []
+    code = []
-    body = [x['content'] for x in anonqueue]
+    for funcname in bb.data.getVar("__BBANONFUNCS", d) or []:
-    flag = { 'python' : 1, 'func' : 1 }
+        code.append("%s(d)" % funcname)
-    bb.data.setVar("__anonfunc", "\n".join(body), d)
+    bb.utils.simple_exec("\n".join(code), {"d": d})
-    bb.data.setVarFlags("__anonfunc", flag, d)
-    from bb import build
-    try:
-        t = bb.data.getVar('T', d)
-        bb.data.setVar('T', '${TMPDIR}/anonfunc/', d)
-        anonfuncs = bb.data.getVar('__BBANONFUNCS', d) or []
-        code = ""
-        for f in anonfuncs:
-            code = code + "    %s(d)\n" % f
-        bb.data.setVar("__anonfunc", code, d)        
-        build.exec_func("__anonfunc", d)
-        bb.data.delVar('T', d)
-        if t:
-            bb.data.setVar('T', t, d)
-    except Exception, e:
-        bb.msg.debug(1, bb.msg.domain.Parsing, "Exception when executing anonymous function: %s" % e)
-        raise
-    bb.data.delVar("__anonqueue", d)
-    bb.data.delVar("__anonfunc", d)
    bb.data.update_data(d)
-    all_handlers = {} 
+    all_handlers = {}
    for var in bb.data.getVar('__BBHANDLERS', d) or []:
        # try to add the handler
-        handler = bb.data.getVar(var,d)
+        handler = bb.data.getVar(var, d)
        bb.event.register(var, handler)
    tasklist = bb.data.getVar('__BBTASKS', d) or []
@@ -360,7 +344,7 @@ def _expand_versions(versions):
    versions = iter(versions)
    while True:
        try:
-            version = versions.next()
+            version = next(versions)
        except StopIteration:
            break
@@ -370,14 +354,14 @@ def _expand_versions(versions):
        else:
            newversions = expand_one(version, int(range_ver.group("from")),
                                     int(range_ver.group("to")))
-            versions = chain(newversions, versions)
+            versions = itertools.chain(newversions, versions)
 def multi_finalize(fn, d):
    safe_d = d
    d = bb.data.createCopy(safe_d)
    try:
-        finalise(fn, d)
+        finalize(fn, d)
    except bb.parse.SkipPackage:
        bb.data.setVar("__SKIPPED", True, d)
    datastores = {"": safe_d}
@@ -420,7 +404,7 @@ def multi_finalize(fn, d):
            d = bb.data.createCopy(safe_d)
            verfunc(pv, d, safe_d)
            try:
-                finalise(fn, d)
+                finalize(fn, d)
            except bb.parse.SkipPackage:
                bb.data.setVar("__SKIPPED", True, d)
@@ -436,15 +420,15 @@ def multi_finalize(fn, d):
        safe_d.setVar("BBCLASSEXTEND", extended)
        _create_variants(datastores, extended.split(), extendfunc)
-    for variant, variant_d in datastores.items():
+    for variant, variant_d in datastores.iteritems():
        if variant:
            try:
-                finalise(fn, variant_d)
+                finalize(fn, variant_d)
            except bb.parse.SkipPackage:
                bb.data.setVar("__SKIPPED", True, variant_d)
    if len(datastores) > 1:
-        variants = filter(None, datastores.keys())
+        variants = filter(None, datastores.iterkeys())
        safe_d.setVar("__VARIANTS", " ".join(variants))
    datastores[""] = d
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index 262c883c95..bb56174881 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -11,7 +11,7 @@
 #  Copyright (C) 2003, 2004  Chris Larson
 #  Copyright (C) 2003, 2004  Phil Blundell
-#   
+#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
@@ -25,15 +25,17 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import re, bb, os, sys, time, string
+from __future__ import absolute_import
+import re, bb, os
 import bb.fetch, bb.build, bb.utils
-from bb import data, fetch
+from bb import data
-from ConfHandler import include, init
+from . import ConfHandler
-from bb.parse import ParseError, resolve_file, ast
+from .. import resolve_file, ast
+from .ConfHandler import include, init
 # For compatibility
-from bb.parse import vars_from_file
+bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
 __func_start_regexp__    = re.compile( r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile( r"inherit\s+(.+)" )
@@ -68,8 +70,8 @@ def inherit(files, d):
    __inherit_cache = data.getVar('__inherit_cache', d) or []
    fn = ""
    lineno = 0
-    files = data.expand(files, d)
    for file in files:
+        file = data.expand(file, d)
        if file[0] != "/" and file[-8:] != ".bbclass":
            file = os.path.join('classes', '%s.bbclass' % file)
@@ -80,17 +82,17 @@ def inherit(files, d):
            include(fn, file, d, "inherit")
            __inherit_cache = data.getVar('__inherit_cache', d) or []
-def get_statements(filename, absolsute_filename, base_name):
+def get_statements(filename, absolute_filename, base_name):
    global cached_statements
    try:
-        return cached_statements[absolsute_filename]
+        return cached_statements[absolute_filename]
    except KeyError:
-        file = open(absolsute_filename, 'r')
+        file = open(absolute_filename, 'r')
        statements = ast.StatementGroup()
        lineno = 0
-        while 1:
+        while True:
            lineno = lineno + 1
            s = file.readline()
            if not s: break
@@ -101,7 +103,7 @@ def get_statements(filename, absolsute_filename, base_name):
            feeder(IN_PYTHON_EOF, "", filename, base_name, statements)
        if filename.endswith(".bbclass") or filename.endswith(".inc"):
-            cached_statements[absolsute_filename] = statements
+            cached_statements[absolute_filename] = statements
        return statements
 def handle(fn, d, include):
@@ -118,7 +120,7 @@ def handle(fn, d, include):
        bb.msg.debug(2, bb.msg.domain.Parsing, "BB " + fn + ": handle(data, include)")
    (root, ext) = os.path.splitext(os.path.basename(fn))
-    base_name = "%s%s" % (root,ext)
+    base_name = "%s%s" % (root, ext)
    init(d)
    if ext == ".bbclass":
@@ -164,7 +166,7 @@ def handle(fn, d, include):
    return d
 def feeder(lineno, s, fn, root, statements):
-    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__,__infunc__, __body__, classes, bb, __residue__
+    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, classes, bb, __residue__
    if __infunc__:
        if s == '}':
            __body__.append('')
@@ -231,10 +233,9 @@ def feeder(lineno, s, fn, root, statements):
        ast.handleInherit(statements, m)
        return
-    from bb.parse import ConfHandler
    return ConfHandler.feeder(lineno, s, fn, statements)
 # Add us to the handlers list
-from bb.parse import handlers
+from .. import handlers
 handlers.append({'supports': supports, 'handle': handle, 'init': init})
 del handlers
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index f4f85de245..9128a2ef8f 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -10,7 +10,7 @@
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
-# 
+#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
@@ -24,7 +24,8 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import re, bb.data, os, sys
+import re, bb.data, os
+import bb.utils
 from bb.parse import ParseError, resolve_file, ast
 #__config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}]+)\s*(?P<colon>:)?(?P<ques>\?)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
@@ -36,10 +37,7 @@ __export_regexp__ = re.compile( r"export\s+(.+)" )
 def init(data):
    topdir = bb.data.getVar('TOPDIR', data)
    if not topdir:
-        topdir = os.getcwd()
+        bb.data.setVar('TOPDIR', os.getcwd(), data)
-        bb.data.setVar('TOPDIR', topdir, data)
-    if not bb.data.getVar('BBPATH', data):
-        bb.fatal("The BBPATH environment variable must be set")
 def supports(fn, d):
@@ -60,7 +58,7 @@ def include(oldfn, fn, data, error_out):
    if not os.path.isabs(fn):
        dname = os.path.dirname(oldfn)
        bbpath = "%s:%s" % (dname, bb.data.getVar("BBPATH", data, 1))
-        abs_fn = bb.which(bbpath, fn)
+        abs_fn = bb.utils.which(bbpath, fn)
        if abs_fn:
            fn = abs_fn
@@ -88,7 +86,7 @@ def handle(fn, data, include):
    statements = ast.StatementGroup()
    lineno = 0
-    while 1:
+    while True:
        lineno = lineno + 1
        s = f.readline()
        if not s: break
diff --git a/bitbake/lib/bb/parse/parse_py/__init__.py b/bitbake/lib/bb/parse/parse_py/__init__.py
index 9e0e00adda..3e658d0de9 100644
--- a/bitbake/lib/bb/parse/parse_py/__init__.py
+++ b/bitbake/lib/bb/parse/parse_py/__init__.py
@@ -25,9 +25,9 @@ File parsers for the BitBake build tools.
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
-__version__ = '1.0'
-__all__ = [ 'ConfHandler', 'BBHandler']
+from __future__ import absolute_import
+from . import ConfHandler
+from . import BBHandler
-import ConfHandler
+__version__ = '1.0'
-import BBHandler
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
index bc4045fe85..df0409cd8a 100644
--- a/bitbake/lib/bb/persist_data.py
+++ b/bitbake/lib/bb/persist_data.py
@@ -16,6 +16,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 import bb, os
+import bb.utils
 try:
    import sqlite3
@@ -33,58 +34,63 @@ class PersistData:
    """
    BitBake Persistent Data Store
-    Used to store data in a central location such that other threads/tasks can 
+    Used to store data in a central location such that other threads/tasks can
    access them at some future date.
-    The "domain" is used as a key to isolate each data pool and in this 
+    The "domain" is used as a key to isolate each data pool and in this
-    implementation corresponds to an SQL table. The SQL table consists of a 
+    implementation corresponds to an SQL table. The SQL table consists of a
    simple key and value pair.
    Why sqlite? It handles all the locking issues for us.
    """
-    def __init__(self, d):
+    def __init__(self, d, persistent_database_connection):
+        if "connection" in persistent_database_connection:
+            self.cursor = persistent_database_connection["connection"].cursor()
+            return
        self.cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
        if self.cachedir in [None, '']:
            bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.")
        try:
            os.stat(self.cachedir)
        except OSError:
-            bb.mkdirhier(self.cachedir)
+            bb.utils.mkdirhier(self.cachedir)
-        self.cachefile = os.path.join(self.cachedir,"bb_persist_data.sqlite3")
+        self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3")
        bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile)
-        self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
+        connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
+        persistent_database_connection["connection"] = connection
+        self.cursor = persistent_database_connection["connection"].cursor()
    def addDomain(self, domain):
        """
        Should be called before any domain is used
        Creates it if it doesn't exist.
        """
-        self.connection.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain)
+        self.cursor.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain)
    def delDomain(self, domain):
        """
        Removes a domain and all the data it contains
        """
-        self.connection.execute("DROP TABLE IF EXISTS %s;" % domain)
+        self.cursor.execute("DROP TABLE IF EXISTS %s;" % domain)
    def getKeyValues(self, domain):
        """
        Return a list of key + value pairs for a domain
        """
        ret = {}
-        data = self.connection.execute("SELECT key, value from %s;" % domain)
+        data = self.cursor.execute("SELECT key, value from %s;" % domain)
        for row in data:
            ret[str(row[0])] = str(row[1])
-        return ret
+        return ret
    def getValue(self, domain, key):
        """
        Return the value of a key for a domain
        """
-        data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key])
+        data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key])
        for row in data:
            return row[1]
@@ -92,7 +98,7 @@ class PersistData:
        """
        Sets the value of a key for a domain
        """
-        data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key])
+        data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key])
        rows = 0
        for row in data:
            rows = rows + 1
@@ -108,14 +114,11 @@ class PersistData:
        self._execute("DELETE from %s where key=?;" % domain, [key])
    def _execute(self, *query):
-        while True:     
+        while True:
            try:
-                self.connection.execute(*query)
+                self.cursor.execute(*query)
                return
-            except sqlite3.OperationalError, e:
+            except sqlite3.OperationalError as e:
                if 'database is locked' in str(e):
                    continue
                raise
-        
-        
diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py
index 058996ba57..58326f0398 100644
--- a/bitbake/lib/bb/providers.py
+++ b/bitbake/lib/bb/providers.py
@@ -62,7 +62,7 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
 def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
    """
    Check if the version pe,pv,pr is the preferred one.
-    If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%' 
+    If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
    """
    if (pr == preferred_r or preferred_r == None):
        if (pe == preferred_e or preferred_e == None):
@@ -103,7 +103,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
        for file_set in pkg_pn:
            for f in file_set:
-                pe,pv,pr = dataCache.pkg_pepvpr[f]
+                pe, pv, pr = dataCache.pkg_pepvpr[f]
                if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
                    preferred_file = f
                    preferred_ver = (pe, pv, pr)
@@ -136,7 +136,7 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
    latest_p = 0
    latest_f = None
    for file_name in file_set:
-        pe,pv,pr = dataCache.pkg_pepvpr[file_name]
+        pe, pv, pr = dataCache.pkg_pepvpr[file_name]
        dp = dataCache.pkg_dp[file_name]
        if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
@@ -169,14 +169,14 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
 def _filterProviders(providers, item, cfgData, dataCache):
    """
-    Take a list of providers and filter/reorder according to the 
+    Take a list of providers and filter/reorder according to the
    environment variables and previous build results
    """
    eligible = []
    preferred_versions = {}
    sortpkg_pn = {}
-    # The order of providers depends on the order of the files on the disk 
+    # The order of providers depends on the order of the files on the disk
    # up to here. Sort pkg_pn to make dependency issues reproducible rather
    # than effectively random.
    providers.sort()
@@ -226,7 +226,7 @@ def _filterProviders(providers, item, cfgData, dataCache):
 def filterProviders(providers, item, cfgData, dataCache):
    """
-    Take a list of providers and filter/reorder according to the 
+    Take a list of providers and filter/reorder according to the
    environment variables and previous build results
    Takes a "normal" target item
    """
@@ -254,7 +254,7 @@ def filterProviders(providers, item, cfgData, dataCache):
 def filterProvidersRunTime(providers, item, cfgData, dataCache):
    """
-    Take a list of providers and filter/reorder according to the 
+    Take a list of providers and filter/reorder according to the
    environment variables and previous build results
    Takes a "runtime" target item
    """
@@ -297,7 +297,7 @@ def getRuntimeProviders(dataCache, rdepend):
    rproviders = []
    if rdepend in dataCache.rproviders:
-       rproviders += dataCache.rproviders[rdepend]
+        rproviders += dataCache.rproviders[rdepend]
    if rdepend in dataCache.packages:
        rproviders += dataCache.packages[rdepend]
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index c5f4380c86..a4aea6c004 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -22,15 +22,15 @@ Handles preparation and execution of a queue of tasks
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-from bb import msg, data, event, mkdirhier, utils
 import bb, os, sys
+from bb import msg, data, event
 import signal
 import stat
 import fcntl
 class TaskFailure(Exception):
    """Exception raised when a task in a runqueue fails"""
-    def __init__(self, x): 
+    def __init__(self, x):
        self.args = x
@@ -60,7 +60,7 @@ class RunQueueStats:
    def taskActive(self):
        self.active = self.active + 1
-# These values indicate the next step due to be run in the 
+# These values indicate the next step due to be run in the
 # runQueue state machine
 runQueuePrepare = 2
 runQueueRunInit = 3
@@ -76,7 +76,7 @@ class RunQueueScheduler:
    """
    def __init__(self, runqueue):
        """
-        The default scheduler just returns the first buildable task (the 
+        The default scheduler just returns the first buildable task (the
        priority map is sorted by task numer)
        """
        self.rq = runqueue
@@ -109,8 +109,7 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
        self.rq = runqueue
-        sortweight = deepcopy(self.rq.runq_weight)
+        sortweight = sorted(deepcopy(self.rq.runq_weight))
-        sortweight.sort()
        copyweight = deepcopy(self.rq.runq_weight)
        self.prio_map = []
@@ -123,10 +122,10 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
 class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
    """
-    A scheduler optimised to complete .bb files are quickly as possible. The 
+    A scheduler optimised to complete .bb files are quickly as possible. The
-    priority map is sorted by task weight, but then reordered so once a given 
+    priority map is sorted by task weight, but then reordered so once a given
    .bb file starts to build, its completed as quickly as possible. This works
-    well where disk space is at a premium and classes like OE's rm_work are in 
+    well where disk space is at a premium and classes like OE's rm_work are in
    force.
    """
    def __init__(self, runqueue):
@@ -135,7 +134,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
        #FIXME - whilst this groups all fnids together it does not reorder the
        #fnid groups optimally.
- 
        basemap = deepcopy(self.prio_map)
        self.prio_map = []
        while (len(basemap) > 0):
@@ -231,7 +230,7 @@ class RunQueue:
                if chain1[index] != chain2[index]:
                    return False
            return True
-            
        def chain_array_contains(chain, chain_array):
            """
            Return True if chain_array contains chain
@@ -286,7 +285,7 @@ class RunQueue:
    def calculate_task_weights(self, endpoints):
        """
-        Calculate a number representing the "weight" of each task. Heavier weighted tasks 
+        Calculate a number representing the "weight" of each task. Heavier weighted tasks
        have more dependencies and hence should be executed sooner for maximum speed.
        This function also sanity checks the task list finding tasks that its not
@@ -307,7 +306,7 @@ class RunQueue:
            weight[listid] = 1
            task_done[listid] = True
-        while 1:
+        while True:
            next_points = []
            for listid in endpoints:
                for revdep in self.runq_depends[listid]:
@@ -318,7 +317,7 @@ class RunQueue:
                        task_done[revdep] = True
            endpoints = next_points
            if len(next_points) == 0:
-                break      
+                break
        # Circular dependency sanity check
        problem_tasks = []
@@ -345,7 +344,7 @@ class RunQueue:
    def prepare_runqueue(self):
        """
-        Turn a set of taskData into a RunQueue and compute data needed 
+        Turn a set of taskData into a RunQueue and compute data needed
        to optimise the execution order.
        """
@@ -365,12 +364,12 @@ class RunQueue:
        # Step A - Work out a list of tasks to run
        #
        # Taskdata gives us a list of possible providers for every build and run
-        # target ordered by priority. It also gives information on each of those 
+        # target ordered by priority. It also gives information on each of those
        # providers.
        #
-        # To create the actual list of tasks to execute we fix the list of 
+        # To create the actual list of tasks to execute we fix the list of
-        # providers and then resolve the dependencies into task IDs. This 
+        # providers and then resolve the dependencies into task IDs. This
-        # process is repeated for each type of dependency (tdepends, deptask, 
+        # process is repeated for each type of dependency (tdepends, deptask,
        # rdeptast, recrdeptask, idepends).
        def add_build_dependencies(depids, tasknames, depends):
@@ -411,12 +410,12 @@ class RunQueue:
            if fnid not in taskData.failed_fnids:
-                # Resolve task internal dependencies 
+                # Resolve task internal dependencies
                #
                # e.g. addtask before X after Y
                depends = taskData.tasks_tdepends[task]
-                # Resolve 'deptask' dependencies 
+                # Resolve 'deptask' dependencies
                #
                # e.g. do_sometask[deptask] = "do_someothertask"
                # (makes sure sometask runs after someothertask of all DEPENDS)
@@ -424,7 +423,7 @@ class RunQueue:
                    tasknames = task_deps['deptask'][taskData.tasks_name[task]].split()
                    add_build_dependencies(taskData.depids[fnid], tasknames, depends)
-                # Resolve 'rdeptask' dependencies 
+                # Resolve 'rdeptask' dependencies
                #
                # e.g. do_sometask[rdeptask] = "do_someothertask"
                # (makes sure sometask runs after someothertask of all RDEPENDS)
@@ -432,7 +431,7 @@ class RunQueue:
                    taskname = task_deps['rdeptask'][taskData.tasks_name[task]]
                    add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends)
-                # Resolve inter-task dependencies 
+                # Resolve inter-task dependencies
                #
                # e.g. do_sometask[depends] = "targetname:do_someothertask"
                # (makes sure sometask runs after targetname's someothertask)
@@ -467,8 +466,8 @@ class RunQueue:
                    newdep = []
                    bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends))
                    for dep in depends:
-                       if task != dep:
+                        if task != dep:
-                          newdep.append(dep)
+                            newdep.append(dep)
                    depends = newdep
            self.runq_fnid.append(taskData.tasks_fnid[task])
@@ -482,7 +481,7 @@ class RunQueue:
        #
        # Build a list of recursive cumulative dependencies for each fnid
        # We do this by fnid, since if A depends on some task in B
-        # we're interested in later tasks B's fnid might have but B itself 
+        # we're interested in later tasks B's fnid might have but B itself
        # doesn't depend on
        #
        # Algorithm is O(tasks) + O(tasks)*O(fnids)
@@ -513,7 +512,7 @@ class RunQueue:
            if len(runq_recrdepends[task]) > 0:
                taskfnid = self.runq_fnid[task]
                for dep in reccumdepends[taskfnid]:
-                    # Ignore self references 
+                    # Ignore self references
                    if dep == task:
                        continue
                    for taskname in runq_recrdepends[task]:
@@ -631,11 +630,11 @@ class RunQueue:
            for dep in revdeps:
                if dep in self.runq_depends[listid]:
                    #self.dump_data(taskData)
-                    bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep] , taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid]))
+                    bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid]))
        bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints))
-        # Calculate task weights 
+        # Calculate task weights
        # Check of higher length circular dependencies
        self.runq_weight = self.calculate_task_weights(endpoints)
@@ -657,7 +656,7 @@ class RunQueue:
            for prov in self.dataCache.fn_provides[fn]:
                if prov not in prov_list:
                    prov_list[prov] = [fn]
-                elif fn not in prov_list[prov]: 
+                elif fn not in prov_list[prov]:
                    prov_list[prov].append(fn)
        error = False
        for prov in prov_list:
@@ -703,7 +702,7 @@ class RunQueue:
                buildable.append(task)
        def check_buildable(self, task, buildable):
-             for revdep in self.runq_revdeps[task]:
+            for revdep in self.runq_revdeps[task]:
                alldeps = 1
                for dep in self.runq_depends[revdep]:
                    if dep in unchecked:
@@ -774,7 +773,7 @@ class RunQueue:
        #print "Not current: %s" % notcurrent
        if len(unchecked) > 0:
-            bb.fatal("check_stamps fatal internal error")
+            bb.msg.fatal(bb.msg.domain.RunQueue, "check_stamps fatal internal error")
        return current
    def check_stamp_task(self, task):
@@ -811,10 +810,10 @@ class RunQueue:
                    try:
                        t2 = os.stat(stampfile2)[stat.ST_MTIME]
                        if t1 < t2:
-                            bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile,stampfile2))
+                            bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2))
                            iscurrent = False
                    except:
-                        bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 ,stampfile))
+                        bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2, stampfile))
                        iscurrent = False
        return iscurrent
@@ -852,7 +851,7 @@ class RunQueue:
            return False
        if self.state is runQueueChildProcess:
-            print "Child process"
+            print("Child process")
            return False
        # Loop
@@ -885,7 +884,7 @@ class RunQueue:
    def task_complete(self, task):
        """
        Mark a task as completed
-        Look at the reverse dependencies and mark any task with 
+        Look at the reverse dependencies and mark any task with
        completed dependencies as buildable
        """
        self.runq_complete[task] = 1
@@ -929,7 +928,7 @@ class RunQueue:
        while True:
            task = None
            if self.stats.active < self.number_tasks:
-                task = self.sched.next()
+                task = next(self.sched)
            if task is not None:
                fn = self.taskData.fn_index[self.runq_fnid[task]]
@@ -948,7 +947,7 @@ class RunQueue:
                try:
                    pipein, pipeout = os.pipe()
                    pid = os.fork() 
-                except OSError, e: 
+                except OSError as e: 
                    bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
                if pid == 0:
                    os.close(pipein)
@@ -982,10 +981,11 @@ class RunQueue:
                    try:
                        self.cooker.tryBuild(fn, taskname[3:])
                    except bb.build.EventException:
-                        bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
                        os._exit(1)
-                    except:
+                    except Exception:
-                        bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
+                        from traceback import format_exc
+                        bb.msg.error(bb.msg.domain.Build, "Build of %s %s failed" % (fn, taskname))
+                        bb.msg.error(bb.msg.domain.Build, format_exc())
                        os._exit(1)
                    os._exit(0)
@@ -1031,12 +1031,13 @@ class RunQueue:
            return
    def finish_runqueue_now(self):
-        bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active)
+        if self.stats.active:
-        for k, v in self.build_pids.iteritems():
+            bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active)
-             try:
+            for k, v in self.build_pids.iteritems():
-                 os.kill(-k, signal.SIGINT)
+                try:
-             except:
+                    os.kill(-k, signal.SIGINT)
-                 pass
+                except:
+                    pass
        for pipe in self.build_pipes:
            self.build_pipes[pipe].read()
@@ -1085,30 +1086,30 @@ class RunQueue:
        """
        bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:")
        for task in range(len(self.runq_task)):
-                bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task, 
+            bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task,
-                        taskQueue.fn_index[self.runq_fnid[task]], 
+                taskQueue.fn_index[self.runq_fnid[task]],
-                        self.runq_task[task], 
+                self.runq_task[task],
-                        self.runq_weight[task], 
+                self.runq_weight[task],
-                        self.runq_depends[task], 
+                self.runq_depends[task],
-                        self.runq_revdeps[task]))
+                self.runq_revdeps[task]))
        bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:")
        for task1 in range(len(self.runq_task)):
            if task1 in self.prio_map:
                task = self.prio_map[task1]
-                bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task, 
+                bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task,
-                        taskQueue.fn_index[self.runq_fnid[task]], 
+                    taskQueue.fn_index[self.runq_fnid[task]],
-                        self.runq_task[task], 
+                    self.runq_task[task],
-                        self.runq_weight[task], 
+                    self.runq_weight[task],
-                        self.runq_depends[task], 
+                    self.runq_depends[task],
-                        self.runq_revdeps[task]))
+                    self.runq_revdeps[task]))
 class TaskFailure(Exception):
    """
    Exception raised when a task in a runqueue fails
    """
-    def __init__(self, x): 
+    def __init__(self, x):
        self.args = x
@@ -1194,6 +1195,5 @@ class runQueuePipe():
        while self.read():
            continue
        if len(self.queue) > 0:
-            print "Warning, worker left partial message"
+            print("Warning, worker left partial message")
        os.close(self.fd)
diff --git a/bitbake/lib/bb/server/none.py b/bitbake/lib/bb/server/none.py
index ebda111582..e28aa8d7d7 100644
--- a/bitbake/lib/bb/server/none.py
+++ b/bitbake/lib/bb/server/none.py
@@ -115,7 +115,7 @@ class BitBakeServer():
    def register_idle_function(self, function, data):
        """Register a function to be called while the server is idle"""
-        assert callable(function)
+        assert hasattr(function, '__call__')
        self._idlefuns[function] = data
    def idle_commands(self, delay):
@@ -178,4 +178,3 @@ class BitBakeServerConnection():
            self.connection.terminateServer()
        except:
            pass
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
index 3364918c77..cb2949fb9f 100644
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -42,7 +42,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import inspect, select
 if sys.hexversion < 0x020600F0:
-    print "Sorry, python 2.6 or later is required for bitbake's XMLRPC mode"
+    print("Sorry, python 2.6 or later is required for bitbake's XMLRPC mode")
    sys.exit(1)
 class BitBakeServerCommands():
@@ -74,7 +74,7 @@ class BitBakeServerCommands():
        Trigger the server to quit
        """
        self.server.quit = True
-        print "Server (cooker) exitting"
+        print("Server (cooker) exitting")
        return
    def ping(self):
@@ -89,8 +89,8 @@ class BitBakeServer(SimpleXMLRPCServer):
    def __init__(self, cooker, interface = ("localhost", 0)):
        """
-        Constructor
+        Constructor
-        """
+        """
        SimpleXMLRPCServer.__init__(self, interface,
                                    requestHandler=SimpleXMLRPCRequestHandler,
                                    logRequests=False, allow_none=True)
@@ -112,7 +112,7 @@ class BitBakeServer(SimpleXMLRPCServer):
    def register_idle_function(self, function, data):
        """Register a function to be called while the server is idle"""
-        assert callable(function)
+        assert hasattr(function, '__call__')
        self._idlefuns[function] = data
    def serve_forever(self):
@@ -146,7 +146,7 @@ class BitBakeServer(SimpleXMLRPCServer):
                    traceback.print_exc()
                    pass
            if nextsleep is None and len(self._idlefuns) > 0:
-                nextsleep = 0  
+                nextsleep = 0
            self.timeout = nextsleep
        # Tell idle functions we're exiting
        for function, data in self._idlefuns.items():
@@ -175,7 +175,7 @@ class BitBakeServerConnection():
    def terminate(self):
        # Don't wait for server indefinitely
        import socket
-        socket.setdefaulttimeout(2) 
+        socket.setdefaulttimeout(2)
        try:
            self.events.system_quit()
        except:
@@ -184,4 +184,3 @@ class BitBakeServerConnection():
            self.connection.terminateServer()
        except:
            pass
diff --git a/bitbake/lib/bb/shell.py b/bitbake/lib/bb/shell.py
index 7abea0f126..f9ca9d5bd3 100644
--- a/bitbake/lib/bb/shell.py
+++ b/bitbake/lib/bb/shell.py
@@ -52,12 +52,14 @@ PROBLEMS:
 # Import and setup global variables
 ##########################################################################
+from __future__ import print_function
+from functools import reduce
 try:
    set
 except NameError:
    from sets import Set as set
-import sys, os, readline, socket, httplib, urllib, commands, popen2, copy, shlex, Queue, fnmatch
+import sys, os, readline, socket, httplib, urllib, commands, popen2, shlex, Queue, fnmatch
-from bb import data, parse, build, fatal, cache, taskdata, runqueue, providers as Providers
+from bb import data, parse, build, cache, taskdata, runqueue, providers as Providers
 __version__ = "0.5.3.1"
 __credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de>
@@ -98,7 +100,7 @@ class BitBakeShellCommands:
    def _checkParsed( self ):
        if not parsed:
-            print "SHELL: This command needs to parse bbfiles..."
+            print("SHELL: This command needs to parse bbfiles...")
            self.parse( None )
    def _findProvider( self, item ):
@@ -119,28 +121,28 @@ class BitBakeShellCommands:
        """Register a new name for a command"""
        new, old = params
        if not old in cmds:
-            print "ERROR: Command '%s' not known" % old
+            print("ERROR: Command '%s' not known" % old)
        else:
            cmds[new] = cmds[old]
-            print "OK"
+            print("OK")
    alias.usage = "<alias> <command>"
    def buffer( self, params ):
        """Dump specified output buffer"""
        index = params[0]
-        print self._shell.myout.buffer( int( index ) )
+        print(self._shell.myout.buffer( int( index ) ))
    buffer.usage = "<index>"
    def buffers( self, params ):
        """Show the available output buffers"""
        commands = self._shell.myout.bufferedCommands()
        if not commands:
-            print "SHELL: No buffered commands available yet. Start doing something."
+            print("SHELL: No buffered commands available yet. Start doing something.")
        else:
-            print "="*35, "Available Output Buffers", "="*27
+            print("="*35, "Available Output Buffers", "="*27)
            for index, cmd in enumerate( commands ):
-                print "| %s %s" % ( str( index ).ljust( 3 ), cmd )
+                print("| %s %s" % ( str( index ).ljust( 3 ), cmd ))
-            print "="*88
+            print("="*88)
    def build( self, params, cmd = "build" ):
        """Build a providee"""
@@ -149,7 +151,7 @@ class BitBakeShellCommands:
        self._checkParsed()
        names = globfilter( cooker.status.pkg_pn, globexpr )
        if len( names ) == 0: names = [ globexpr ]
-        print "SHELL: Building %s" % ' '.join( names )
+        print("SHELL: Building %s" % ' '.join( names ))
        td = taskdata.TaskData(cooker.configuration.abort)
        localdata = data.createCopy(cooker.configuration.data)
@@ -168,22 +170,22 @@ class BitBakeShellCommands:
                tasks.append([name, "do_%s" % cmd])
            td.add_unresolved(localdata, cooker.status)
-            
            rq = runqueue.RunQueue(cooker, localdata, cooker.status, td, tasks)
            rq.prepare_runqueue()
            rq.execute_runqueue()
        except Providers.NoProvider:
-            print "ERROR: No Provider"
+            print("ERROR: No Provider")
            last_exception = Providers.NoProvider
-        except runqueue.TaskFailure, fnids:
+        except runqueue.TaskFailure as fnids:
            for fnid in fnids:
-                print "ERROR: '%s' failed" % td.fn_index[fnid]
+                print("ERROR: '%s' failed" % td.fn_index[fnid])
            last_exception = runqueue.TaskFailure
-        except build.EventException, e:
+        except build.EventException as e:
-            print "ERROR: Couldn't build '%s'" % names
+            print("ERROR: Couldn't build '%s'" % names)
            last_exception = e
@@ -216,7 +218,7 @@ class BitBakeShellCommands:
        if bbfile is not None:
            os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) )
        else:
-            print "ERROR: Nothing provides '%s'" % name
+            print("ERROR: Nothing provides '%s'" % name)
    edit.usage = "<providee>"
    def environment( self, params ):
@@ -239,14 +241,14 @@ class BitBakeShellCommands:
        global last_exception
        name = params[0]
        bf = completeFilePath( name )
-        print "SHELL: Calling '%s' on '%s'" % ( cmd, bf )
+        print("SHELL: Calling '%s' on '%s'" % ( cmd, bf ))
        try:
            cooker.buildFile(bf, cmd)
        except parse.ParseError:
-            print "ERROR: Unable to open or parse '%s'" % bf
+            print("ERROR: Unable to open or parse '%s'" % bf)
-        except build.EventException, e:
+        except build.EventException as e:
-            print "ERROR: Couldn't build '%s'" % name
+            print("ERROR: Couldn't build '%s'" % name)
            last_exception = e
    fileBuild.usage = "<bbfile>"
@@ -270,62 +272,62 @@ class BitBakeShellCommands:
    def fileReparse( self, params ):
        """(re)Parse a bb file"""
        bbfile = params[0]
-        print "SHELL: Parsing '%s'" % bbfile
+        print("SHELL: Parsing '%s'" % bbfile)
        parse.update_mtime( bbfile )
        cooker.bb_cache.cacheValidUpdate(bbfile)
        fromCache = cooker.bb_cache.loadData(bbfile, cooker.configuration.data, cooker.status)
        cooker.bb_cache.sync()
        if False: #fromCache:
-            print "SHELL: File has not been updated, not reparsing"
+            print("SHELL: File has not been updated, not reparsing")
        else:
-            print "SHELL: Parsed"
+            print("SHELL: Parsed")
    fileReparse.usage = "<bbfile>"
    def abort( self, params ):
        """Toggle abort task execution flag (see bitbake -k)"""
        cooker.configuration.abort = not cooker.configuration.abort
-        print "SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort )
+        print("SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort ))
    def force( self, params ):
        """Toggle force task execution flag (see bitbake -f)"""
        cooker.configuration.force = not cooker.configuration.force
-        print "SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force )
+        print("SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force ))
    def help( self, params ):
        """Show a comprehensive list of commands and their purpose"""
-        print "="*30, "Available Commands", "="*30
+        print("="*30, "Available Commands", "="*30)
        for cmd in sorted(cmds):
-            function,numparams,usage,helptext = cmds[cmd]
+            function, numparams, usage, helptext = cmds[cmd]
-            print "| %s | %s" % (usage.ljust(30), helptext)
+            print("| %s | %s" % (usage.ljust(30), helptext))
-        print "="*78
+        print("="*78)
    def lastError( self, params ):
        """Show the reason or log that was produced by the last BitBake event exception"""
        if last_exception is None:
-            print "SHELL: No Errors yet (Phew)..."
+            print("SHELL: No Errors yet (Phew)...")
        else:
            reason, event = last_exception.args
-            print "SHELL: Reason for the last error: '%s'" % reason
+            print("SHELL: Reason for the last error: '%s'" % reason)
            if ':' in reason:
                msg, filename = reason.split( ':' )
                filename = filename.strip()
-                print "SHELL: Dumping log file for last error:"
+                print("SHELL: Dumping log file for last error:")
                try:
-                    print open( filename ).read()
+                    print(open( filename ).read())
                except IOError:
-                    print "ERROR: Couldn't open '%s'" % filename
+                    print("ERROR: Couldn't open '%s'" % filename)
    def match( self, params ):
        """Dump all files or providers matching a glob expression"""
        what, globexpr = params
        if what == "files":
            self._checkParsed()
-            for key in globfilter( cooker.status.pkg_fn, globexpr ): print key
+            for key in globfilter( cooker.status.pkg_fn, globexpr ): print(key)
        elif what == "providers":
            self._checkParsed()
-            for key in globfilter( cooker.status.pkg_pn, globexpr ): print key
+            for key in globfilter( cooker.status.pkg_pn, globexpr ): print(key)
        else:
-            print "Usage: match %s" % self.print_.usage
+            print("Usage: match %s" % self.print_.usage)
    match.usage = "<files|providers> <glob>"
    def new( self, params ):
@@ -335,15 +337,15 @@ class BitBakeShellCommands:
        fulldirname = "%s/%s" % ( packages, dirname )
        if not os.path.exists( fulldirname ):
-            print "SHELL: Creating '%s'" % fulldirname
+            print("SHELL: Creating '%s'" % fulldirname)
            os.mkdir( fulldirname )
        if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ):
            if os.path.exists( "%s/%s" % ( fulldirname, filename ) ):
-                print "SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename )
+                print("SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename ))
                return False
-            print "SHELL: Creating '%s/%s'" % ( fulldirname, filename )
+            print("SHELL: Creating '%s/%s'" % ( fulldirname, filename ))
            newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" )
-            print >>newpackage,"""DESCRIPTION = ""
+            print("""DESCRIPTION = ""
 SECTION = ""
 AUTHOR = ""
 HOMEPAGE = ""
@@ -370,7 +372,7 @@ SRC_URI = ""
 #do_install() {
 #
 #}
-"""
+""", file=newpackage)
            newpackage.close()
            os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) )
    new.usage = "<directory> <filename>"
@@ -390,14 +392,14 @@ SRC_URI = ""
    def pasteLog( self, params ):
        """Send the last event exception error log (if there is one) to http://rafb.net/paste"""
        if last_exception is None:
-            print "SHELL: No Errors yet (Phew)..."
+            print("SHELL: No Errors yet (Phew)...")
        else:
            reason, event = last_exception.args
-            print "SHELL: Reason for the last error: '%s'" % reason
+            print("SHELL: Reason for the last error: '%s'" % reason)
            if ':' in reason:
                msg, filename = reason.split( ':' )
                filename = filename.strip()
-                print "SHELL: Pasting log file to pastebin..."
+                print("SHELL: Pasting log file to pastebin...")
                file = open( filename ).read()
                sendToPastebin( "contents of " + filename, file )
@@ -419,23 +421,23 @@ SRC_URI = ""
        cooker.buildDepgraph()
        global parsed
        parsed = True
-        print
+        print()
    def reparse( self, params ):
        """(re)Parse a providee's bb file"""
        bbfile = self._findProvider( params[0] )
        if bbfile is not None:
-            print "SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] )
+            print("SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] ))
            self.fileReparse( [ bbfile ] )
        else:
-            print "ERROR: Nothing provides '%s'" % params[0]
+            print("ERROR: Nothing provides '%s'" % params[0])
    reparse.usage = "<providee>"
    def getvar( self, params ):
        """Dump the contents of an outer BitBake environment variable"""
        var = params[0]
        value = data.getVar( var, cooker.configuration.data, 1 )
-        print value
+        print(value)
    getvar.usage = "<variable>"
    def peek( self, params ):
@@ -445,9 +447,9 @@ SRC_URI = ""
        if bbfile is not None:
            the_data = cooker.bb_cache.loadDataFull(bbfile, cooker.configuration.data)
            value = the_data.getVar( var, 1 )
-            print value
+            print(value)
        else:
-            print "ERROR: Nothing provides '%s'" % name
+            print("ERROR: Nothing provides '%s'" % name)
    peek.usage = "<providee> <variable>"
    def poke( self, params ):
@@ -455,7 +457,7 @@ SRC_URI = ""
        name, var, value = params
        bbfile = self._findProvider( name )
        if bbfile is not None:
-            print "ERROR: Sorry, this functionality is currently broken"
+            print("ERROR: Sorry, this functionality is currently broken")
            #d = cooker.pkgdata[bbfile]
            #data.setVar( var, value, d )
@@ -463,7 +465,7 @@ SRC_URI = ""
            #cooker.pkgdata.setDirty(bbfile, d)
            #print "OK"
        else:
-            print "ERROR: Nothing provides '%s'" % name
+            print("ERROR: Nothing provides '%s'" % name)
    poke.usage = "<providee> <variable> <value>"
    def print_( self, params ):
@@ -471,12 +473,12 @@ SRC_URI = ""
        what = params[0]
        if what == "files":
            self._checkParsed()
-            for key in cooker.status.pkg_fn: print key
+            for key in cooker.status.pkg_fn: print(key)
        elif what == "providers":
            self._checkParsed()
-            for key in cooker.status.providers: print key
+            for key in cooker.status.providers: print(key)
        else:
-            print "Usage: print %s" % self.print_.usage
+            print("Usage: print %s" % self.print_.usage)
    print_.usage = "<files|providers>"
    def python( self, params ):
@@ -496,7 +498,7 @@ SRC_URI = ""
        """Set an outer BitBake environment variable"""
        var, value = params
        data.setVar( var, value, cooker.configuration.data )
-        print "OK"
+        print("OK")
    setVar.usage = "<variable> <value>"
    def rebuild( self, params ):
@@ -508,7 +510,7 @@ SRC_URI = ""
    def shell( self, params ):
        """Execute a shell command and dump the output"""
        if params != "":
-            print commands.getoutput( " ".join( params ) )
+            print(commands.getoutput( " ".join( params ) ))
    shell.usage = "<...>"
    def stage( self, params ):
@@ -518,17 +520,17 @@ SRC_URI = ""
    def status( self, params ):
        """<just for testing>"""
-        print "-" * 78
+        print("-" * 78)
-        print "building list = '%s'" % cooker.building_list
+        print("building list = '%s'" % cooker.building_list)
-        print "build path = '%s'" % cooker.build_path
+        print("build path = '%s'" % cooker.build_path)
-        print "consider_msgs_cache = '%s'" % cooker.consider_msgs_cache
+        print("consider_msgs_cache = '%s'" % cooker.consider_msgs_cache)
-        print "build stats = '%s'" % cooker.stats
+        print("build stats = '%s'" % cooker.stats)
-        if last_exception is not None: print "last_exception = '%s'" % repr( last_exception.args )
+        if last_exception is not None: print("last_exception = '%s'" % repr( last_exception.args ))
-        print "memory output contents = '%s'" % self._shell.myout._buffer
+        print("memory output contents = '%s'" % self._shell.myout._buffer)
    def test( self, params ):
        """<just for testing>"""
-        print "testCommand called with '%s'" % params
+        print("testCommand called with '%s'" % params)
    def unpack( self, params ):
        """Execute 'unpack' on a providee"""
@@ -553,12 +555,12 @@ SRC_URI = ""
        try:
            providers = cooker.status.providers[item]
        except KeyError:
-            print "SHELL: ERROR: Nothing provides", preferred
+            print("SHELL: ERROR: Nothing provides", preferred)
        else:
            for provider in providers:
                if provider == pf: provider = " (***) %s" % provider
                else:              provider = "       %s" % provider
-                print provider
+                print(provider)
    which.usage = "<providee>"
 ##########################################################################
@@ -583,7 +585,7 @@ def sendToPastebin( desc, content ):
    mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" )
    mydata["text"] = content
    params = urllib.urlencode( mydata )
-    headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
+    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
    host = "rafb.net"
    conn = httplib.HTTPConnection( "%s:80" % host )
@@ -594,9 +596,9 @@ def sendToPastebin( desc, content ):
    if response.status == 302:
        location = response.getheader( "location" ) or "unknown"
-        print "SHELL: Pasted to http://%s%s" % ( host, location )
+        print("SHELL: Pasted to http://%s%s" % ( host, location ))
    else:
-        print "ERROR: %s %s" % ( response.status, response.reason )
+        print("ERROR: %s %s" % ( response.status, response.reason ))
 def completer( text, state ):
    """Return a possible readline completion"""
@@ -643,7 +645,7 @@ def columnize( alist, width = 80 ):
    return reduce(lambda line, word, width=width: '%s%s%s' %
                  (line,
                   ' \n'[(len(line[line.rfind('\n')+1:])
-                         + len(word.split('\n',1)[0]
+                         + len(word.split('\n', 1)[0]
                              ) >= width)],
                   word),
                  alist
@@ -718,7 +720,7 @@ class BitBakeShell:
        except IOError:
            pass  # It doesn't exist yet.
-        print __credits__
+        print(__credits__)
    def cleanup( self ):
        """Write readline history and clean up resources"""
@@ -726,7 +728,7 @@ class BitBakeShell:
        try:
            readline.write_history_file( self.historyfilename )
        except:
-            print "SHELL: Unable to save command history"
+            print("SHELL: Unable to save command history")
    def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ):
        """Register a command"""
@@ -740,11 +742,11 @@ class BitBakeShell:
        try:
            function, numparams, usage, helptext = cmds[command]
        except KeyError:
-            print "SHELL: ERROR: '%s' command is not a valid command." % command
+            print("SHELL: ERROR: '%s' command is not a valid command." % command)
            self.myout.removeLast()
        else:
            if (numparams != -1) and (not len( params ) == numparams):
-                print "Usage: '%s'" % usage
+                print("Usage: '%s'" % usage)
                return
            result = function( self.commands, params )
@@ -759,7 +761,7 @@ class BitBakeShell:
                if not cmdline:
                    continue
                if "|" in cmdline:
-                    print "ERROR: '|' in startup file is not allowed. Ignoring line"
+                    print("ERROR: '|' in startup file is not allowed. Ignoring line")
                    continue
                self.commandQ.put( cmdline.strip() )
@@ -801,10 +803,10 @@ class BitBakeShell:
                                sys.stdout.write( pipe.fromchild.read() )
                        #
            except EOFError:
-                print
+                print()
                return
            except KeyboardInterrupt:
-                print
+                print()
 ##########################################################################
 # Start function - called from the BitBake command line utility
@@ -819,4 +821,4 @@ def start( aCooker ):
    bbshell.cleanup()
 if __name__ == "__main__":
-    print "SHELL: Sorry, this program should only be called by BitBake."
+    print("SHELL: Sorry, this program should only be called by BitBake.")
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 3e5e006f5f..e31f967850 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -34,7 +34,7 @@ def re_match_strings(target, strings):
    for name in strings:
        if (name==target or
-                re.search(name,target)!=None):
+                re.search(name, target)!=None):
            return True
    return False
@@ -84,7 +84,7 @@ class TaskData:
    def getrun_id(self, name):
        """
-        Return an ID number for the run target name. 
+        Return an ID number for the run target name.
        If it doesn't exist, create one.
        """
        if not name in self.run_names_index:
@@ -95,7 +95,7 @@ class TaskData:
    def getfn_id(self, name):
        """
-        Return an ID number for the filename. 
+        Return an ID number for the filename.
        If it doesn't exist, create one.
        """
        if not name in self.fn_index:
@@ -271,7 +271,7 @@ class TaskData:
    def get_unresolved_build_targets(self, dataCache):
        """
-        Return a list of build targets who's providers 
+        Return a list of build targets who's providers
        are unknown.
        """
        unresolved = []
@@ -286,7 +286,7 @@ class TaskData:
    def get_unresolved_run_targets(self, dataCache):
        """
-        Return a list of runtime targets who's providers 
+        Return a list of runtime targets who's providers
        are unknown.
        """
        unresolved = []
@@ -304,7 +304,7 @@ class TaskData:
        Return a list of providers of item
        """
        targetid = self.getbuild_id(item)
-   
        return self.build_targets[targetid]
    def get_dependees(self, itemid):
@@ -354,20 +354,15 @@ class TaskData:
            self.add_provider_internal(cfgData, dataCache, item)
        except bb.providers.NoProvider:
            if self.abort:
-                if self.get_rdependees_str(item):
-                    bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (item, self.get_dependees_str(item)))
-                else:
-                    bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (item))
                raise
-            targetid = self.getbuild_id(item)
+            self.remove_buildtarget(self.getbuild_id(item))
-            self.remove_buildtarget(targetid)
        self.mark_external_target(item)
    def add_provider_internal(self, cfgData, dataCache, item):
        """
        Add the providers of item to the task data
-        Mark entries were specifically added externally as against dependencies 
+        Mark entries were specifically added externally as against dependencies
        added internally during dependency resolution
        """
@@ -375,11 +370,7 @@ class TaskData:
            return
        if not item in dataCache.providers:
-            if self.get_rdependees_str(item):
+            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_rdependees_str(item)), cfgData)
-                bb.msg.note(2, bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (item, self.get_dependees_str(item)))
-            else:
-                bb.msg.note(2, bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (item))
-            bb.event.fire(bb.event.NoProvider(item), cfgData)
            raise bb.providers.NoProvider(item)
        if self.have_build_target(item):
@@ -391,8 +382,7 @@ class TaskData:
        eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
        if not eligible:
-            bb.msg.note(2, bb.msg.domain.Provider, "No buildable provider PROVIDES '%s' but '%s' DEPENDS on or otherwise requires it. Enable debugging and see earlier logs to find unbuildable providers." % (item, self.get_dependees_str(item)))
+            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item)), cfgData)
-            bb.event.fire(bb.event.NoProvider(item), cfgData)
            raise bb.providers.NoProvider(item)
        if len(eligible) > 1 and foundUnique == False:
@@ -400,8 +390,6 @@ class TaskData:
                providers_list = []
                for fn in eligible:
                    providers_list.append(dataCache.pkg_fn[fn])
-                bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
-                bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
                bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
            self.consider_msgs_cache.append(item)
@@ -431,16 +419,14 @@ class TaskData:
        all_p = bb.providers.getRuntimeProviders(dataCache, item)
        if not all_p:
-            bb.msg.error(bb.msg.domain.Provider, "'%s' RDEPENDS/RRECOMMENDS or otherwise requires the runtime entity '%s' but it wasn't found in any PACKAGE or RPROVIDES variables" % (self.get_rdependees_str(item), item))
+            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item)), cfgData)
-            bb.event.fire(bb.event.NoProvider(item, runtime=True), cfgData)
            raise bb.providers.NoRProvider(item)
        eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
        eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
        if not eligible:
-            bb.msg.error(bb.msg.domain.Provider, "'%s' RDEPENDS/RRECOMMENDS or otherwise requires the runtime entity '%s' but it wasn't found in any PACKAGE or RPROVIDES variables of any buildable targets.\nEnable debugging and see earlier logs to find unbuildable targets." % (self.get_rdependees_str(item), item))
+            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item)), cfgData)
-            bb.event.fire(bb.event.NoProvider(item, runtime=True), cfgData)
            raise bb.providers.NoRProvider(item)
        if len(eligible) > 1 and numberPreferred == 0:
@@ -448,9 +434,7 @@ class TaskData:
                providers_list = []
                for fn in eligible:
                    providers_list.append(dataCache.pkg_fn[fn])
-                bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
+                bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
-                bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
-                bb.event.fire(bb.event.MultipleProviders(item,providers_list, runtime=True), cfgData)
            self.consider_msgs_cache.append(item)
        if numberPreferred > 1:
@@ -458,9 +442,7 @@ class TaskData:
                providers_list = []
                for fn in eligible:
                    providers_list.append(dataCache.pkg_fn[fn])
-                bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (top %s entries preferred) (%s);" % (item, numberPreferred, ", ".join(providers_list)))
+                bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
-                bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
-                bb.event.fire(bb.event.MultipleProviders(item,providers_list, runtime=True), cfgData)
            self.consider_msgs_cache.append(item)
        # run through the list until we find one that we can build
@@ -515,8 +497,9 @@ class TaskData:
                    self.fail_fnid(self.tasks_fnid[taskid], missing_list)
        if self.abort and targetid in self.external_targets:
-            bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (self.build_names_index[targetid], missing_list))
+            target = self.build_names_index[targetid]
-            raise bb.providers.NoProvider
+            bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (target, missing_list))
+            raise bb.providers.NoProvider(target)
    def remove_runtarget(self, targetid, missing_list = []):
        """
@@ -539,7 +522,7 @@ class TaskData:
        Resolve all unresolved build and runtime targets
        """
        bb.msg.note(1, bb.msg.domain.TaskData, "Resolving any missing task queue dependencies")
-        while 1:
+        while True:
            added = 0
            for target in self.get_unresolved_build_targets(dataCache):
                try:
@@ -548,10 +531,6 @@ class TaskData:
                except bb.providers.NoProvider:
                    targetid = self.getbuild_id(target)
                    if self.abort and targetid in self.external_targets:
-                        if self.get_rdependees_str(target):
-                            bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (target, self.get_dependees_str(target)))
-                        else:
-                            bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (target))
                        raise
                    self.remove_buildtarget(targetid)
            for target in self.get_unresolved_run_targets(dataCache):
@@ -594,9 +573,9 @@ class TaskData:
        bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
        for task in range(len(self.tasks_name)):
            bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
-                task, 
+                task,
-                self.fn_index[self.tasks_fnid[task]], 
+                self.fn_index[self.tasks_fnid[task]],
-                self.tasks_name[task], 
+                self.tasks_name[task],
                self.tasks_tdepends[task]))
        bb.msg.debug(3, bb.msg.domain.TaskData, "dependency ids (per fn):")
@@ -606,5 +585,3 @@ class TaskData:
        bb.msg.debug(3, bb.msg.domain.TaskData, "runtime dependency ids (per fn):")
        for fnid in self.rdepids:
            bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
diff --git a/bitbake/lib/bb/ui/__init__.py b/bitbake/lib/bb/ui/__init__.py
index c6a377a8e6..a4805ed028 100644
--- a/bitbake/lib/bb/ui/__init__.py
+++ b/bitbake/lib/bb/ui/__init__.py
@@ -15,4 +15,3 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
diff --git a/bitbake/lib/bb/ui/crumbs/__init__.py b/bitbake/lib/bb/ui/crumbs/__init__.py
index c6a377a8e6..a4805ed028 100644
--- a/bitbake/lib/bb/ui/crumbs/__init__.py
+++ b/bitbake/lib/bb/ui/crumbs/__init__.py
@@ -15,4 +15,3 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
diff --git a/bitbake/lib/bb/ui/crumbs/buildmanager.py b/bitbake/lib/bb/ui/crumbs/buildmanager.py
index f89e8eefd4..e858d75e4c 100644
--- a/bitbake/lib/bb/ui/crumbs/buildmanager.py
+++ b/bitbake/lib/bb/ui/crumbs/buildmanager.py
@@ -28,7 +28,7 @@ import time
 class BuildConfiguration:
    """ Represents a potential *or* historic *or* concrete build. It
    encompasses all the things that we need to tell bitbake to do to make it
-    build what we want it to build. 
+    build what we want it to build.
    It also stored the metadata URL and the set of possible machines (and the
    distros / images / uris for these. Apart from the metdata URL these are
@@ -73,34 +73,33 @@ class BuildConfiguration:
        return self.urls
    # It might be a lot lot better if we stored these in like, bitbake conf
-    # file format. 
+    # file format.
-    @staticmethod 
+    @staticmethod
    def load_from_file (filename):
-        f = open (filename, "r")
        conf = BuildConfiguration()
-        for line in f.readlines():
+        with open(filename, "r") as f:
-            data = line.split (";")[1]
+            for line in f:
-            if (line.startswith ("metadata-url;")):
+                data = line.split (";")[1]
-                conf.metadata_url = data.strip()
+                if (line.startswith ("metadata-url;")):
-                continue
+                    conf.metadata_url = data.strip()
-            if (line.startswith ("url;")):
+                    continue
-                conf.urls += [data.strip()]
+                if (line.startswith ("url;")):
-                continue
+                    conf.urls += [data.strip()]
-            if (line.startswith ("extra-url;")):
+                    continue
-                conf.extra_urls += [data.strip()]
+                if (line.startswith ("extra-url;")):
-                continue
+                    conf.extra_urls += [data.strip()]
-            if (line.startswith ("machine;")):
+                    continue
-                conf.machine = data.strip()
+                if (line.startswith ("machine;")):
-                continue
+                    conf.machine = data.strip()
-            if (line.startswith ("distribution;")):
+                    continue
-                conf.distro = data.strip()
+                if (line.startswith ("distribution;")):
-                continue
+                    conf.distro = data.strip()
-            if (line.startswith ("image;")):
+                    continue
-                conf.image = data.strip()
+                if (line.startswith ("image;")):
-                continue
+                    conf.image = data.strip()
+                    continue
-        f.close ()
        return conf
    # Serialise to a file. This is part of the build process and we use this
@@ -140,13 +139,13 @@ class BuildResult(gobject.GObject):
    ".conf" in the directory for the build.
    This is GObject so that it can be included in the TreeStore."""
-    
    (STATE_COMPLETE, STATE_FAILED, STATE_ONGOING) = \
        (0, 1, 2)
    def __init__ (self, parent, identifier):
        gobject.GObject.__init__ (self)
-        self.date = None 
+        self.date = None
        self.files = []
        self.status = None
@@ -157,8 +156,8 @@ class BuildResult(gobject.GObject):
        # format build-<year><month><day>-<ordinal> we can easily
        # pull it out.
        # TODO: Better to stat a file?
-        (_ , date, revision) = identifier.split ("-")
+        (_, date, revision) = identifier.split ("-")
-        print date
+        print(date)
        year = int (date[0:4])
        month = int (date[4:6])
@@ -181,7 +180,7 @@ class BuildResult(gobject.GObject):
                self.add_file (file)
    def add_file (self, file):
-        # Just add the file for now. Don't care about the type. 
+        # Just add the file for now. Don't care about the type.
        self.files += [(file, None)]
 class BuildManagerModel (gtk.TreeStore):
@@ -194,7 +193,7 @@ class BuildManagerModel (gtk.TreeStore):
    def __init__ (self):
        gtk.TreeStore.__init__ (self,
-            gobject.TYPE_STRING, 
+            gobject.TYPE_STRING,
            gobject.TYPE_STRING,
            gobject.TYPE_STRING,
            gobject.TYPE_STRING,
@@ -207,7 +206,7 @@ class BuildManager (gobject.GObject):
    "results" directory but is also used for starting a new build."""
    __gsignals__ = {
-        'population-finished' : (gobject.SIGNAL_RUN_LAST, 
+        'population-finished' : (gobject.SIGNAL_RUN_LAST,
                                 gobject.TYPE_NONE,
                                 ()),
        'populate-error' : (gobject.SIGNAL_RUN_LAST,
@@ -220,13 +219,13 @@ class BuildManager (gobject.GObject):
        date = long (time.mktime (result.date.timetuple()))
        # Add a top level entry for the build
-        
-        self.model.set (iter, 
+        self.model.set (iter,
            BuildManagerModel.COL_IDENT, result.identifier,
            BuildManagerModel.COL_DESC, result.conf.image,
-            BuildManagerModel.COL_MACHINE, result.conf.machine, 
+            BuildManagerModel.COL_MACHINE, result.conf.machine,
-            BuildManagerModel.COL_DISTRO, result.conf.distro, 
+            BuildManagerModel.COL_DISTRO, result.conf.distro,
-            BuildManagerModel.COL_BUILD_RESULT, result, 
+            BuildManagerModel.COL_BUILD_RESULT, result,
            BuildManagerModel.COL_DATE, date,
            BuildManagerModel.COL_STATE, result.state)
@@ -257,7 +256,7 @@ class BuildManager (gobject.GObject):
        while (iter):
            (ident, state) = self.model.get(iter,
-                BuildManagerModel.COL_IDENT, 
+                BuildManagerModel.COL_IDENT,
                BuildManagerModel.COL_STATE)
            if state == BuildResult.STATE_ONGOING:
@@ -385,8 +384,8 @@ class BuildManager (gobject.GObject):
                build_directory])
            server.runCommand(["buildTargets", [conf.image], "rootfs"])
-        except Exception, e:
+        except Exception as e:
-            print e
+            print(e)
 class BuildManagerTreeView (gtk.TreeView):
    """ The tree view for the build manager. This shows the historic builds
@@ -422,29 +421,29 @@ class BuildManagerTreeView (gtk.TreeView):
        # Misc descriptiony thing
        renderer = gtk.CellRendererText ()
-        col = gtk.TreeViewColumn (None, renderer, 
+        col = gtk.TreeViewColumn (None, renderer,
            text=BuildManagerModel.COL_DESC)
        self.append_column (col)
        # Machine
        renderer = gtk.CellRendererText ()
-        col = gtk.TreeViewColumn ("Machine", renderer, 
+        col = gtk.TreeViewColumn ("Machine", renderer,
            text=BuildManagerModel.COL_MACHINE)
        self.append_column (col)
        # distro
        renderer = gtk.CellRendererText ()
-        col = gtk.TreeViewColumn ("Distribution", renderer, 
+        col = gtk.TreeViewColumn ("Distribution", renderer,
            text=BuildManagerModel.COL_DISTRO)
        self.append_column (col)
        # date (using a custom function for formatting the cell contents it
        # takes epoch -> human readable string)
        renderer = gtk.CellRendererText ()
-        col = gtk.TreeViewColumn ("Date", renderer, 
+        col = gtk.TreeViewColumn ("Date", renderer,
            text=BuildManagerModel.COL_DATE)
        self.append_column (col)
-        col.set_cell_data_func (renderer, 
+        col.set_cell_data_func (renderer,
            self.date_format_custom_cell_data_func)
        # For status.
@@ -454,4 +453,3 @@ class BuildManagerTreeView (gtk.TreeView):
        self.append_column (col)
        col.set_cell_data_func (renderer,
            self.state_format_custom_cell_data_fun)
diff --git a/bitbake/lib/bb/ui/crumbs/runningbuild.py b/bitbake/lib/bb/ui/crumbs/runningbuild.py
index 18afd6674d..b4416ecbb3 100644
--- a/bitbake/lib/bb/ui/crumbs/runningbuild.py
+++ b/bitbake/lib/bb/ui/crumbs/runningbuild.py
@@ -24,7 +24,7 @@ import gobject
 class RunningBuildModel (gtk.TreeStore):
    (COL_TYPE, COL_PACKAGE, COL_TASK, COL_MESSAGE, COL_ICON, COL_ACTIVE) = (0, 1, 2, 3, 4, 5)
    def __init__ (self):
-        gtk.TreeStore.__init__ (self, 
+        gtk.TreeStore.__init__ (self,
                                gobject.TYPE_STRING,
                                gobject.TYPE_STRING,
                                gobject.TYPE_STRING,
@@ -34,7 +34,7 @@ class RunningBuildModel (gtk.TreeStore):
 class RunningBuild (gobject.GObject):
    __gsignals__ = {
-          'build-succeeded' : (gobject.SIGNAL_RUN_LAST, 
+          'build-succeeded' : (gobject.SIGNAL_RUN_LAST,
                               gobject.TYPE_NONE,
                               ()),
          'build-failed' : (gobject.SIGNAL_RUN_LAST,
@@ -63,7 +63,7 @@ class RunningBuild (gobject.GObject):
        # for the message.
        if hasattr(event, 'pid'):
            pid = event.pid
-            if self.pids_to_task.has_key(pid):
+            if pid in self.pids_to_task:
                (package, task) = self.pids_to_task[pid]
                parent = self.tasks_to_iter[(package, task)]
@@ -82,29 +82,29 @@ class RunningBuild (gobject.GObject):
            # Add the message to the tree either at the top level if parent is
            # None otherwise as a descendent of a task.
-            self.model.append (parent, 
+            self.model.append (parent,
                               (event.__name__.split()[-1], # e.g. MsgWarn, MsgError
-                                package, 
+                                package,
                                task,
                                event._message,
-                                icon, 
+                                icon,
                                False))
        elif isinstance(event, bb.build.TaskStarted):
            (package, task) = (event._package, event._task)
            # Save out this PID.
-            self.pids_to_task[pid] = (package,task)
+            self.pids_to_task[pid] = (package, task)
            # Check if we already have this package in our model. If so then
            # that can be the parent for the task. Otherwise we create a new
            # top level for the package.
-            if (self.tasks_to_iter.has_key ((package, None))):
+            if ((package, None) in self.tasks_to_iter):
                parent = self.tasks_to_iter[(package, None)]
            else:
-                parent = self.model.append (None, (None, 
+                parent = self.model.append (None, (None,
-                                                   package, 
+                                                   package,
                                                   None,
-                                                   "Package: %s" % (package), 
+                                                   "Package: %s" % (package),
                                                   None,
                                                   False))
                self.tasks_to_iter[(package, None)] = parent
@@ -114,10 +114,10 @@ class RunningBuild (gobject.GObject):
            self.model.set(parent, self.model.COL_ICON, "gtk-execute")
            # Add an entry in the model for this task
-            i = self.model.append (parent, (None, 
+            i = self.model.append (parent, (None,
-                                            package, 
+                                            package,
                                            task,
-                                            "Task: %s" % (task), 
+                                            "Task: %s" % (task),
                                            None,
                                            False))
@@ -176,5 +176,3 @@ class RunningBuildTreeView (gtk.TreeView):
        renderer = gtk.CellRendererText ()
        col = gtk.TreeViewColumn ("Message", renderer, text=3)
        self.append_column (col)
diff --git a/bitbake/lib/bb/ui/depexp.py b/bitbake/lib/bb/ui/depexp.py
index cfa5b6564e..1cd58cac18 100644
--- a/bitbake/lib/bb/ui/depexp.py
+++ b/bitbake/lib/bb/ui/depexp.py
@@ -201,14 +201,14 @@ def init(server, eventHandler):
    try:
        cmdline = server.runCommand(["getCmdLineAction"])
        if not cmdline or cmdline[0] != "generateDotGraph":
-            print "This UI is only compatible with the -g option"
+            print("This UI is only compatible with the -g option")
            return
        ret = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]])
        if ret != True:
-            print "Couldn't run command! %s" % ret
+            print("Couldn't run command! %s" % ret)
            return
-    except xmlrpclib.Fault, x:
+    except xmlrpclib.Fault as x:
-        print "XMLRPC Fault getting commandline:\n %s" % x
+        print("XMLRPC Fault getting commandline:\n %s" % x)
        return
    shutdown = 0
@@ -233,8 +233,8 @@ def init(server, eventHandler):
                x = event.sofar
                y = event.total
                if x == y:
-                    print("\nParsing finished. %d cached, %d parsed, %d skipped, %d masked, %d errors." 
+                    print(("\nParsing finished. %d cached, %d parsed, %d skipped, %d masked, %d errors."
-                        % ( event.cached, event.parsed, event.skipped, event.masked, event.errors))
+                        % ( event.cached, event.parsed, event.skipped, event.masked, event.errors)))
                    pbar.hide()
                gtk.gdk.threads_enter()
                pbar.progress.set_fraction(float(x)/float(y))
@@ -250,7 +250,7 @@ def init(server, eventHandler):
            if isinstance(event, bb.command.CookerCommandCompleted):
                continue
            if isinstance(event, bb.command.CookerCommandFailed):
-                print "Command execution failed: %s" % event.error
+                print("Command execution failed: %s" % event.error)
                break
            if isinstance(event, bb.cooker.CookerExit):
                break
@@ -259,14 +259,13 @@ def init(server, eventHandler):
        except KeyboardInterrupt:
            if shutdown == 2:
-                print "\nThird Keyboard Interrupt, exit.\n"
+                print("\nThird Keyboard Interrupt, exit.\n")
                break
            if shutdown == 1:
-                print "\nSecond Keyboard Interrupt, stopping...\n"
+                print("\nSecond Keyboard Interrupt, stopping...\n")
                server.runCommand(["stateStop"])
            if shutdown == 0:
-                print "\nKeyboard Interrupt, closing down...\n"
+                print("\nKeyboard Interrupt, closing down...\n")
                server.runCommand(["stateShutdown"])
            shutdown = shutdown + 1
            pass
diff --git a/bitbake/lib/bb/ui/goggle.py b/bitbake/lib/bb/ui/goggle.py
index 94995d82db..2cfa002f8a 100644
--- a/bitbake/lib/bb/ui/goggle.py
+++ b/bitbake/lib/bb/ui/goggle.py
@@ -25,13 +25,13 @@ from bb.ui.crumbs.runningbuild import RunningBuildTreeView, RunningBuild
 def event_handle_idle_func (eventHandler, build):
-  # Consume as many messages as we can in the time available to us
+    # Consume as many messages as we can in the time available to us
-  event = eventHandler.getEvent()
+    event = eventHandler.getEvent()
-  while event:
+    while event:
-      build.handle_event (event)
+        build.handle_event (event)
-      event = eventHandler.getEvent()
+        event = eventHandler.getEvent()
-  return True
+    return True
 class MainWindow (gtk.Window):
    def __init__ (self):
@@ -55,15 +55,15 @@ def init (server, eventHandler):
    window.cur_build_tv.set_model (running_build.model)
    try:
        cmdline = server.runCommand(["getCmdLineAction"])
-        print cmdline
+        print(cmdline)
        if not cmdline:
            return 1
        ret = server.runCommand(cmdline)
        if ret != True:
-            print "Couldn't get default commandline! %s" % ret
+            print("Couldn't get default commandline! %s" % ret)
            return 1
-    except xmlrpclib.Fault, x:
+    except xmlrpclib.Fault as x:
-        print "XMLRPC Fault getting commandline:\n %s" % x
+        print("XMLRPC Fault getting commandline:\n %s" % x)
        return 1
    # Use a timeout function for probing the event queue to find out if we
@@ -74,4 +74,3 @@ def init (server, eventHandler):
                         running_build)
    gtk.main()
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index ed26bb2b4c..f81759abf8 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -18,8 +18,9 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import os
+from __future__ import division
+import os
 import sys
 import itertools
 import xmlrpclib
@@ -44,10 +45,10 @@ def init(server, eventHandler):
            return 1
        ret = server.runCommand(cmdline)
        if ret != True:
-            print "Couldn't get default commandline! %s" % ret
+            print("Couldn't get default commandline! %s" % ret)
            return 1
-    except xmlrpclib.Fault, x:
+    except xmlrpclib.Fault as x:
-        print "XMLRPC Fault getting commandline:\n %s" % x
+        print("XMLRPC Fault getting commandline:\n %s" % x)
        return 1
    shutdown = 0
@@ -65,39 +66,39 @@ def init(server, eventHandler):
            if shutdown and helper.needUpdate:
                activetasks, failedtasks = helper.getTasks()
                if activetasks:
-                    print "Waiting for %s active tasks to finish:" % len(activetasks)
+                    print("Waiting for %s active tasks to finish:" % len(activetasks))
                    tasknum = 1
                    for task in activetasks:
-                        print "%s: %s (pid %s)" % (tasknum, activetasks[task]["title"], task)
+                        print("%s: %s (pid %s)" % (tasknum, activetasks[task]["title"], task))
                        tasknum = tasknum + 1
            if isinstance(event, bb.msg.MsgPlain):
-                print event._message
+                print(event._message)
                continue
            if isinstance(event, bb.msg.MsgDebug):
-                print 'DEBUG: ' + event._message
+                print('DEBUG: ' + event._message)
                continue
            if isinstance(event, bb.msg.MsgNote):
-                print 'NOTE: ' + event._message
+                print('NOTE: ' + event._message)
                continue
            if isinstance(event, bb.msg.MsgWarn):
-                print 'WARNING: ' + event._message
+                print('WARNING: ' + event._message)
                continue
            if isinstance(event, bb.msg.MsgError):
                return_value = 1
-                print 'ERROR: ' + event._message
+                print('ERROR: ' + event._message)
                continue
            if isinstance(event, bb.msg.MsgFatal):
                return_value = 1
-                print 'FATAL: ' + event._message
+                print('FATAL: ' + event._message)
                break
            if isinstance(event, bb.build.TaskFailed):
                return_value = 1
                logfile = event.logfile
-                if logfile:
+                if logfile and os.path.exists(logfile):
-                    print "ERROR: Logfile of failure stored in: %s" % logfile
+                    print("ERROR: Logfile of failure stored in: %s" % logfile)
                    if 1 or includelogs:
-                        print "Log data follows:"
+                        print("Log data follows:")
                        f = open(logfile, "r")
                        lines = []
                        while True:
@@ -110,19 +111,19 @@ def init(server, eventHandler):
                                if len(lines) > int(loglines):
                                    lines.pop(0)
                            else:
-                                print '| %s' % l
+                                print('| %s' % l)
                        f.close()
                        if lines:
                            for line in lines:
-                                print line
+                                print(line)
            if isinstance(event, bb.build.TaskBase):
-                print "NOTE: %s" % event._message
+                print("NOTE: %s" % event._message)
                continue
            if isinstance(event, bb.event.ParseProgress):
                x = event.sofar
                y = event.total
                if os.isatty(sys.stdout.fileno()):
-                    sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
+                    sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( next(parsespin), x, y, x*100//y ) )
                    sys.stdout.flush()
                else:
                    if x == 1:
@@ -132,8 +133,8 @@ def init(server, eventHandler):
                        sys.stdout.write("done.")
                        sys.stdout.flush()
                if x == y:
-                    print("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." 
+                    print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
-                        % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))
+                        % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                continue
            if isinstance(event, bb.command.CookerCommandCompleted):
@@ -143,39 +144,48 @@ def init(server, eventHandler):
                continue
            if isinstance(event, bb.command.CookerCommandFailed):
                return_value = 1
-                print "Command execution failed: %s" % event.error
+                print("Command execution failed: %s" % event.error)
                break
            if isinstance(event, bb.cooker.CookerExit):
                break
-            # ignore
-            if isinstance(event, bb.event.BuildStarted):
-                continue
-            if isinstance(event, bb.event.BuildCompleted):
-                continue
            if isinstance(event, bb.event.MultipleProviders):
+                print("NOTE: multiple providers are available for %s%s (%s)" % (event._is_runtime and "runtime " or "",
+                                                                                event._item,
+                                                                                ", ".join(event._candidates)))
+                print("NOTE: consider defining a PREFERRED_PROVIDER entry to match %s" % event._item)
+                continue
+            if isinstance(event, bb.event.NoProvider):
+                if event._runtime:
+                    r = "R"
+                else:
+                    r = ""
+                if event._dependees:
+                    print("ERROR: Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r))
+                else:
+                    print("ERROR: Nothing %sPROVIDES '%s'" % (r, event._item))
                continue
-            if isinstance(event, bb.runqueue.runQueueEvent):
-                continue
+            # ignore
-            if isinstance(event, bb.runqueue.runQueueExitWait):
+            if isinstance(event, (bb.event.BuildBase,
-                continue
+                                  bb.event.StampUpdate,
-            if isinstance(event, bb.event.StampUpdate):
+                                  bb.event.ConfigParsed,
-                continue
+                                  bb.event.RecipeParsed,
-            if isinstance(event, bb.event.ConfigParsed):
+                                  bb.runqueue.runQueueEvent,
-                continue
+                                  bb.runqueue.runQueueExitWait)):
-            if isinstance(event, bb.event.RecipeParsed):
                continue
-            print "Unknown Event: %s" % event
+            print("Unknown Event: %s" % event)
        except KeyboardInterrupt:
            if shutdown == 2:
-                print "\nThird Keyboard Interrupt, exit.\n"
+                print("\nThird Keyboard Interrupt, exit.\n")
                break
            if shutdown == 1:
-                print "\nSecond Keyboard Interrupt, stopping...\n"
+                print("\nSecond Keyboard Interrupt, stopping...\n")
                server.runCommand(["stateStop"])
            if shutdown == 0:
-                print "\nKeyboard Interrupt, closing down...\n"
+                print("\nKeyboard Interrupt, closing down...\n")
                server.runCommand(["stateShutdown"])
            shutdown = shutdown + 1
            pass
diff --git a/bitbake/lib/bb/ui/ncurses.py b/bitbake/lib/bb/ui/ncurses.py
index 14310dc124..3fed4c58a8 100644
--- a/bitbake/lib/bb/ui/ncurses.py
+++ b/bitbake/lib/bb/ui/ncurses.py
@@ -44,6 +44,8 @@
 """
+from __future__ import division
 import os, sys, curses, itertools, time
 import bb
 import xmlrpclib
@@ -136,7 +138,7 @@ class NCursesUI:
        """Thread Activity Window"""
        def __init__( self, x, y, width, height ):
            NCursesUI.DecoratedWindow.__init__( self, "Thread Activity", x, y, width, height )
-    
        def setStatus( self, thread, text ):
            line = "%02d: %s" % ( thread, text )
            width = self.dimensions[WIDTH]
@@ -199,8 +201,8 @@ class NCursesUI:
        main_left = 0
        main_top = 0
-        main_height = ( height / 3 * 2 )
+        main_height = ( height // 3 * 2 )
-        main_width = ( width / 3 ) * 2
+        main_width = ( width // 3 ) * 2
        clo_left = main_left
        clo_top = main_top + main_height
        clo_height = height - main_height - main_top - 1
@@ -225,17 +227,17 @@ class NCursesUI:
        helper = uihelper.BBUIHelper()
        shutdown = 0
-   
        try:
            cmdline = server.runCommand(["getCmdLineAction"])
            if not cmdline:
                return
            ret = server.runCommand(cmdline)
            if ret != True:
-                print "Couldn't get default commandlind! %s" % ret
+                print("Couldn't get default commandlind! %s" % ret)
                return
-        except xmlrpclib.Fault, x:
+        except xmlrpclib.Fault as x:
-            print "XMLRPC Fault getting commandline:\n %s" % x
+            print("XMLRPC Fault getting commandline:\n %s" % x)
            return
        exitflag = False
@@ -246,7 +248,7 @@ class NCursesUI:
                    continue
                helper.eventHandler(event)
                #mw.appendText("%s\n" % event[0])
-                if isinstance(event, bb.build.Task):
+                if isinstance(event, bb.build.TaskBase):
                    mw.appendText("NOTE: %s\n" % event._message)
                if isinstance(event, bb.msg.MsgDebug):
                    mw.appendText('DEBUG: ' + event._message + '\n')
@@ -263,10 +265,10 @@ class NCursesUI:
                    y = event.total
                    if x == y:
                        mw.setStatus("Idle")
-                        mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked." 
+                        mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked."
                                % ( event.cached, event.parsed, event.skipped, event.masked ))
                    else:
-                        mw.setStatus("Parsing: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
+                        mw.setStatus("Parsing: %s (%04d/%04d) [%2d %%]" % ( next(parsespin), x, y, x*100//y ) )
 #                if isinstance(event, bb.build.TaskFailed):
 #                    if event.logfile:
 #                        if data.getVar("BBINCLUDELOGS", d):
@@ -301,12 +303,12 @@ class NCursesUI:
                    taw.setText(0, 0, "")
                    if activetasks:
                        taw.appendText("Active Tasks:\n")
-                        for task in activetasks:
+                        for task in activetasks.itervalues():
-                            taw.appendText(task)
+                            taw.appendText(task["title"])
                    if failedtasks:
                        taw.appendText("Failed Tasks:\n")
                        for task in failedtasks:
-                            taw.appendText(task)
+                            taw.appendText(task["title"])
                curses.doupdate()
            except KeyboardInterrupt:
@@ -324,7 +326,7 @@ class NCursesUI:
 def init(server, eventHandler):
    if not os.isatty(sys.stdout.fileno()):
-        print "FATAL: Unable to run 'ncurses' UI without a TTY."
+        print("FATAL: Unable to run 'ncurses' UI without a TTY.")
        return
    ui = NCursesUI()
    try:
@@ -332,4 +334,3 @@ def init(server, eventHandler):
    except:
        import traceback
        traceback.print_exc()
diff --git a/bitbake/lib/bb/ui/puccho.py b/bitbake/lib/bb/ui/puccho.py
index 713aa1f4a6..a627fc803f 100644
--- a/bitbake/lib/bb/ui/puccho.py
+++ b/bitbake/lib/bb/ui/puccho.py
@@ -24,6 +24,7 @@ import gtk.glade
 import threading
 import urllib2
 import os
+import contextlib
 from bb.ui.crumbs.buildmanager import BuildManager, BuildConfiguration
 from bb.ui.crumbs.buildmanager import BuildManagerTreeView
@@ -38,7 +39,7 @@ class MetaDataLoader(gobject.GObject):
    on what machines are available. The distribution and images available for
    the machine and the the uris to use for building the given machine."""
    __gsignals__ = {
-        'success' : (gobject.SIGNAL_RUN_LAST, 
+        'success' : (gobject.SIGNAL_RUN_LAST,
                     gobject.TYPE_NONE,
                     ()),
        'error' : (gobject.SIGNAL_RUN_LAST,
@@ -77,20 +78,19 @@ class MetaDataLoader(gobject.GObject):
        def run (self):
            result = {}
            try:
-                f = urllib2.urlopen (self.url)
+                with contextlib.closing (urllib2.urlopen (self.url)) as f:
+                    # Parse the metadata format. The format is....
-                # Parse the metadata format. The format is....
+                    # <machine>;<default distro>|<distro>...;<default image>|<image>...;<type##url>|...
-                # <machine>;<default distro>|<distro>...;<default image>|<image>...;<type##url>|...
+                    for line in f:
-                for line in f.readlines():
+                        components = line.split(";")
-                    components = line.split(";")
+                        if (len (components) < 4):
-                    if (len (components) < 4):
+                            raise MetaDataLoader.LoaderThread.LoaderImportException
-                        raise MetaDataLoader.LoaderThread.LoaderImportException
+                        machine = components[0]
-                    machine = components[0]
+                        distros = components[1].split("|")
-                    distros = components[1].split("|")
+                        images = components[2].split("|")
-                    images = components[2].split("|")
+                        urls = components[3].split("|")
-                    urls = components[3].split("|")
+                        result[machine] = (distros, images, urls)
-                    result[machine] = (distros, images, urls)
                # Create an object representing this *potential*
                # configuration. It can become concrete if the machine, distro
@@ -104,13 +104,13 @@ class MetaDataLoader(gobject.GObject):
                gobject.idle_add (MetaDataLoader.emit_success_signal,
                    self.loader)
-            except MetaDataLoader.LoaderThread.LoaderImportException, e:
+            except MetaDataLoader.LoaderThread.LoaderImportException as e:
                gobject.idle_add (MetaDataLoader.emit_error_signal, self.loader,
                    "Repository metadata corrupt")
-            except Exception, e:
+            except Exception as e:
                gobject.idle_add (MetaDataLoader.emit_error_signal, self.loader,
                    "Unable to download repository metadata")
-                print e
+                print(e)
    def try_fetch_from_url (self, url):
        # Try and download the metadata. Firing a signal if successful
@@ -211,7 +211,7 @@ class BuildSetupDialog (gtk.Dialog):
        # Build
        button = gtk.Button ("_Build", None, True)
        image = gtk.Image ()
-        image.set_from_stock (gtk.STOCK_EXECUTE,gtk.ICON_SIZE_BUTTON)
+        image.set_from_stock (gtk.STOCK_EXECUTE, gtk.ICON_SIZE_BUTTON)
        button.set_image (image)
        self.add_action_widget (button, BuildSetupDialog.RESPONSE_BUILD)
        button.show_all ()
@@ -293,7 +293,7 @@ class BuildSetupDialog (gtk.Dialog):
        if (active_iter):
            self.configuration.machine = model.get(active_iter, 0)[0]
-        # Extract the chosen distro from the combo 
+        # Extract the chosen distro from the combo
        model = self.distribution_combo.get_model()
        active_iter = self.distribution_combo.get_active_iter()
        if (active_iter):
@@ -311,62 +311,62 @@ class BuildSetupDialog (gtk.Dialog):
 #
 # TODO: Should be a method on the RunningBuild class
 def event_handle_timeout (eventHandler, build):
-  # Consume as many messages as we can ...
+    # Consume as many messages as we can ...
-  event = eventHandler.getEvent()
+    event = eventHandler.getEvent()
-  while event:
+    while event:
-      build.handle_event (event)
+        build.handle_event (event)
-      event = eventHandler.getEvent()
+        event = eventHandler.getEvent()
-  return True
+    return True
 class MainWindow (gtk.Window):
-  # Callback that gets fired when the user hits a button in the
+    # Callback that gets fired when the user hits a button in the
-  # BuildSetupDialog.
+    # BuildSetupDialog.
-  def build_dialog_box_response_cb (self, dialog, response_id):
+    def build_dialog_box_response_cb (self, dialog, response_id):
-      conf = None
+        conf = None
-      if (response_id == BuildSetupDialog.RESPONSE_BUILD):
+        if (response_id == BuildSetupDialog.RESPONSE_BUILD):
-          dialog.update_configuration()
+            dialog.update_configuration()
-          print dialog.configuration.machine, dialog.configuration.distro, \
+            print(dialog.configuration.machine, dialog.configuration.distro, \
-              dialog.configuration.image
+                dialog.configuration.image)
-          conf = dialog.configuration
+            conf = dialog.configuration
-      dialog.destroy()
+        dialog.destroy()
-      if conf:
+        if conf:
-          self.manager.do_build (conf)
+            self.manager.do_build (conf)
-  def build_button_clicked_cb (self, button):
+    def build_button_clicked_cb (self, button):
-    dialog = BuildSetupDialog ()
+        dialog = BuildSetupDialog ()
-    # For some unknown reason Dialog.run causes nice little deadlocks ... :-(
+        # For some unknown reason Dialog.run causes nice little deadlocks ... :-(
-    dialog.connect ("response", self.build_dialog_box_response_cb)
+        dialog.connect ("response", self.build_dialog_box_response_cb)
-    dialog.show()
+        dialog.show()
-  def __init__ (self):
+    def __init__ (self):
-      gtk.Window.__init__ (self)
+        gtk.Window.__init__ (self)
-      # Pull in *just* the main vbox from the Glade XML data and then pack
+        # Pull in *just* the main vbox from the Glade XML data and then pack
-      # that inside the window
+        # that inside the window
-      gxml = gtk.glade.XML (os.path.dirname(__file__) + "/crumbs/puccho.glade",
+        gxml = gtk.glade.XML (os.path.dirname(__file__) + "/crumbs/puccho.glade",
-          root = "main_window_vbox")
+            root = "main_window_vbox")
-      vbox = gxml.get_widget ("main_window_vbox")
+        vbox = gxml.get_widget ("main_window_vbox")
-      self.add (vbox)
+        self.add (vbox)
-      # Create the tree views for the build manager view and the progress view
+        # Create the tree views for the build manager view and the progress view
-      self.build_manager_view = BuildManagerTreeView()
+        self.build_manager_view = BuildManagerTreeView()
-      self.running_build_view = RunningBuildTreeView()
+        self.running_build_view = RunningBuildTreeView()
-      # Grab the scrolled windows that we put the tree views into
+        # Grab the scrolled windows that we put the tree views into
-      self.results_scrolledwindow = gxml.get_widget ("results_scrolledwindow")
+        self.results_scrolledwindow = gxml.get_widget ("results_scrolledwindow")
-      self.progress_scrolledwindow = gxml.get_widget ("progress_scrolledwindow")
+        self.progress_scrolledwindow = gxml.get_widget ("progress_scrolledwindow")
-      # Put the tree views inside ...
+        # Put the tree views inside ...
-      self.results_scrolledwindow.add (self.build_manager_view)
+        self.results_scrolledwindow.add (self.build_manager_view)
-      self.progress_scrolledwindow.add (self.running_build_view)
+        self.progress_scrolledwindow.add (self.running_build_view)
-      # Hook up the build button...
+        # Hook up the build button...
-      self.build_button = gxml.get_widget ("main_toolbutton_build")
+        self.build_button = gxml.get_widget ("main_toolbutton_build")
-      self.build_button.connect ("clicked", self.build_button_clicked_cb)
+        self.build_button.connect ("clicked", self.build_button_clicked_cb)
 # I'm not very happy about the current ownership of the RunningBuild. I have
 # my suspicions that this object should be held by the BuildManager since we
@@ -383,11 +383,11 @@ def running_build_succeeded_cb (running_build, manager):
    # BuildManager. It can then hook onto the signals directly and drive
    # interesting things it cares about.
    manager.notify_build_succeeded ()
-    print "build succeeded"
+    print("build succeeded")
 def running_build_failed_cb (running_build, manager):
    # As above
-    print "build failed"
+    print("build failed")
    manager.notify_build_failed ()
 def init (server, eventHandler):
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index 36302f4da7..f1e4d791ee 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -19,7 +19,7 @@
 """
-Use this class to fork off a thread to recieve event callbacks from the bitbake 
+Use this class to fork off a thread to recieve event callbacks from the bitbake
 server and queue them for the UI to process. This process must be used to avoid
 client/server deadlocks.
 """
@@ -110,16 +110,15 @@ class UIXMLRPCServer (SimpleXMLRPCServer):
                return (sock, addr)
            except socket.timeout:
                pass
-        return (None,None)
+        return (None, None)
    def close_request(self, request):
        if request is None:
            return
        SimpleXMLRPCServer.close_request(self, request)
-        
    def process_request(self, request, client_address):
        if request is None:
            return
        SimpleXMLRPCServer.process_request(self, request, client_address)
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 86b9c724ed..c0cc9c6ea2 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -19,10 +19,22 @@ BitBake Utility Functions
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import re, fcntl, os, string, stat, shutil, time
+import sys
+import bb
+import errno
+import bb.msg
+from commands import getstatusoutput
+# Version comparison
 separators = ".-"
-import re, fcntl, os, types, bb, string, stat, shutil
+# Context used in better_exec, eval
-from commands import getstatusoutput
+_context = {
+    "os": os,
+    "bb": bb,
+    "time": time,
+}
 def explode_version(s):
    r = []
@@ -60,9 +72,9 @@ def vercmp_part(a, b):
        if ca == None and cb == None:
            return 0
-        if type(ca) is types.StringType:
+        if isinstance(ca, basestring):
            sa = ca in separators
-        if type(cb) is types.StringType:
+        if isinstance(cb, basestring):
            sb = cb in separators
        if sa and not sb:
            return -1
@@ -85,6 +97,131 @@ def vercmp(ta, tb):
        r = vercmp_part(ra, rb)
    return r
+_package_weights_ = {"pre":-2, "p":0, "alpha":-4, "beta":-3, "rc":-1}    # dicts are unordered
+_package_ends_ = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ]            # so we need ordered list
+def relparse(myver):
+    """Parses the last elements of a version number into a triplet, that can
+    later be compared.
+    """
+    number = 0
+    p1 = 0
+    p2 = 0
+    mynewver = myver.split('_')
+    if len(mynewver) == 2:
+        # an _package_weights_
+        number = float(mynewver[0])
+        match = 0
+        for x in _package_ends_:
+            elen = len(x)
+            if mynewver[1][:elen] == x:
+                match = 1
+                p1 = _package_weights_[x]
+                try:
+                    p2 = float(mynewver[1][elen:])
+                except:
+                    p2 = 0
+                break
+        if not match:
+            # normal number or number with letter at end
+            divider = len(myver)-1
+            if myver[divider:] not in "1234567890":
+                # letter at end
+                p1 = ord(myver[divider:])
+                number = float(myver[0:divider])
+            else:
+                number = float(myver)
+    else:
+        # normal number or number with letter at end
+        divider = len(myver)-1
+        if myver[divider:] not in "1234567890":
+            #letter at end
+            p1 = ord(myver[divider:])
+            number = float(myver[0:divider])
+        else:
+            number = float(myver)
+    return [number, p1, p2]
+__vercmp_cache__ = {}
+def vercmp_string(val1, val2):
+    """This takes two version strings and returns an integer to tell you whether
+    the versions are the same, val1>val2 or val2>val1.
+    """
+    # quick short-circuit
+    if val1 == val2:
+        return 0
+    valkey = val1 + " " + val2
+    # cache lookup
+    try:
+        return __vercmp_cache__[valkey]
+        try:
+            return - __vercmp_cache__[val2 + " " + val1]
+        except KeyError:
+            pass
+    except KeyError:
+        pass
+    # consider 1_p2 vc 1.1
+    # after expansion will become (1_p2,0) vc (1,1)
+    # then 1_p2 is compared with 1 before 0 is compared with 1
+    # to solve the bug we need to convert it to (1,0_p2)
+    # by splitting _prepart part and adding it back _after_expansion
+    val1_prepart = val2_prepart = ''
+    if val1.count('_'):
+        val1, val1_prepart = val1.split('_', 1)
+    if val2.count('_'):
+        val2, val2_prepart = val2.split('_', 1)
+    # replace '-' by '.'
+    # FIXME: Is it needed? can val1/2 contain '-'?
+    val1 = val1.split("-")
+    if len(val1) == 2:
+        val1[0] = val1[0] + "." + val1[1]
+    val2 = val2.split("-")
+    if len(val2) == 2:
+        val2[0] = val2[0] + "." + val2[1]
+    val1 = val1[0].split('.')
+    val2 = val2[0].split('.')
+    # add back decimal point so that .03 does not become "3" !
+    for x in range(1, len(val1)):
+        if val1[x][0] == '0' :
+            val1[x] = '.' + val1[x]
+    for x in range(1, len(val2)):
+        if val2[x][0] == '0' :
+            val2[x] = '.' + val2[x]
+    # extend varion numbers
+    if len(val2) < len(val1):
+        val2.extend(["0"]*(len(val1)-len(val2)))
+    elif len(val1) < len(val2):
+        val1.extend(["0"]*(len(val2)-len(val1)))
+    # add back _prepart tails
+    if val1_prepart:
+        val1[-1] += '_' + val1_prepart
+    if val2_prepart:
+        val2[-1] += '_' + val2_prepart
+    # The above code will extend version numbers out so they
+    # have the same number of digits.
+    for x in range(0, len(val1)):
+        cmp1 = relparse(val1[x])
+        cmp2 = relparse(val2[x])
+        for y in range(0, 3):
+            myret = cmp1[y] - cmp2[y]
+            if myret != 0:
+                __vercmp_cache__[valkey] = myret
+                return myret
+    __vercmp_cache__[valkey] = 0
+    return 0
 def explode_deps(s):
    """
    Take an RDEPENDS style string of format:
@@ -154,26 +291,22 @@ def _print_trace(body, line):
    """
    Print the Environment of a Text Body
    """
-    import bb
    # print the environment of the method
    bb.msg.error(bb.msg.domain.Util, "Printing the environment of the function")
-    min_line = max(1,line-4)
+    min_line = max(1, line-4)
-    max_line = min(line+4,len(body)-1)
+    max_line = min(line + 4, len(body)-1)
-    for i in range(min_line,max_line+1):
+    for i in range(min_line, max_line + 1):
        bb.msg.error(bb.msg.domain.Util, "\t%.4d:%s" % (i, body[i-1]) )
-def better_compile(text, file, realfile):
+def better_compile(text, file, realfile, mode = "exec"):
    """
    A better compile method. This method
    will print  the offending lines.
    """
    try:
-        return compile(text, file, "exec")
+        return compile(text, file, mode)
-    except Exception, e:
+    except Exception as e:
-        import bb,sys
        # split the text into lines again
        body = text.split('\n')
        bb.msg.error(bb.msg.domain.Util, "Error in compiling python function in: ", realfile)
@@ -191,18 +324,18 @@ def better_exec(code, context, text, realfile):
    print the lines that are responsible for the
    error.
    """
-    import bb,sys
+    import bb.parse
    try:
-        exec code in context
+        exec(code, _context, context)
    except:
-        (t,value,tb) = sys.exc_info()
+        (t, value, tb) = sys.exc_info()
        if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
            raise
        # print the Header of the Error Message
        bb.msg.error(bb.msg.domain.Util, "Error in executing python function in: %s" % realfile)
-        bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t,value) )
+        bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t, value))
        # let us find the line number now
        while tb.tb_next:
@@ -212,48 +345,14 @@ def better_exec(code, context, text, realfile):
        line = traceback.tb_lineno(tb)
        _print_trace( text.split('\n'), line )
-        
        raise
-def Enum(*names):
+def simple_exec(code, context):
-   """
+    exec(code, _context, context)
-   A simple class to give Enum support
-   """
+def better_eval(source, locals):
+    return eval(source, _context, locals)
-   assert names, "Empty enums are not supported"
-   class EnumClass(object):
-      __slots__ = names
-      def __iter__(self):        return iter(constants)
-      def __len__(self):         return len(constants)
-      def __getitem__(self, i):  return constants[i]
-      def __repr__(self):        return 'Enum' + str(names)
-      def __str__(self):         return 'enum ' + str(constants)
-   class EnumValue(object):
-      __slots__ = ('__value')
-      def __init__(self, value): self.__value = value
-      Value = property(lambda self: self.__value)
-      EnumType = property(lambda self: EnumType)
-      def __hash__(self):        return hash(self.__value)
-      def __cmp__(self, other):
-         # C fans might want to remove the following assertion
-         # to make all enums comparable by ordinal value {;))
-         assert self.EnumType is other.EnumType, "Only values from the same enum are comparable"
-         return cmp(self.__value, other.__value)
-      def __invert__(self):      return constants[maximum - self.__value]
-      def __nonzero__(self):     return bool(self.__value)
-      def __repr__(self):        return str(names[self.__value])
-   maximum = len(names) - 1
-   constants = [None] * len(names)
-   for i, each in enumerate(names):
-      val = EnumValue(i)
-      setattr(EnumClass, each, val)
-      constants[i] = val
-   constants = tuple(constants)
-   EnumType = EnumClass()
-   return EnumType
 def lockfile(name):
    """
@@ -262,37 +361,36 @@ def lockfile(name):
    """
    path = os.path.dirname(name)
    if not os.path.isdir(path):
-        import bb, sys
        bb.msg.error(bb.msg.domain.Util, "Error, lockfile path does not exist!: %s" % path)
        sys.exit(1)
    while True:
        # If we leave the lockfiles lying around there is no problem
        # but we should clean up after ourselves. This gives potential
-        # for races though. To work around this, when we acquire the lock 
+        # for races though. To work around this, when we acquire the lock
-        # we check the file we locked was still the lock file on disk. 
+        # we check the file we locked was still the lock file on disk.
-        # by comparing inode numbers. If they don't match or the lockfile 
+        # by comparing inode numbers. If they don't match or the lockfile
        # no longer exists, we start again.
-        # This implementation is unfair since the last person to request the 
+        # This implementation is unfair since the last person to request the
        # lock is the most likely to win it.
        try:
-            lf = open(name, "a+")
+            lf = open(name, "a + ")
            fcntl.flock(lf.fileno(), fcntl.LOCK_EX)
            statinfo = os.fstat(lf.fileno())
            if os.path.exists(lf.name):
-               statinfo2 = os.stat(lf.name)
+                statinfo2 = os.stat(lf.name)
-               if statinfo.st_ino == statinfo2.st_ino:
+                if statinfo.st_ino == statinfo2.st_ino:
-                   return lf
+                    return lf
            # File no longer exists or changed, retry
            lf.close
-        except Exception, e:
+        except Exception as e:
            continue
 def unlockfile(lf):
    """
-    Unlock a file locked using lockfile()                               
+    Unlock a file locked using lockfile()
    """
    os.unlink(lf.name)
    fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
@@ -308,7 +406,7 @@ def md5_file(filename):
    except ImportError:
        import md5
        m = md5.new()
-    
    for line in open(filename):
        m.update(line)
    return m.hexdigest()
@@ -368,19 +466,17 @@ def filter_environment(good_vars):
    are not known and may influence the build in a negative way.
    """
-    import bb
    removed_vars = []
    for key in os.environ.keys():
        if key in good_vars:
            continue
-        
        removed_vars.append(key)
        os.unsetenv(key)
        del os.environ[key]
    if len(removed_vars):
-        bb.debug(1, "Removed the following variables from the environment:", ",".join(removed_vars))
+        bb.msg.debug(1, bb.msg.domain.Util, "Removed the following variables from the environment:", ",".join(removed_vars))
    return removed_vars
@@ -410,7 +506,7 @@ def build_environment(d):
    """
    Build an environment from all exported variables.
    """
-    import bb
+    import bb.data
    for var in bb.data.keys(d):
        export = bb.data.getVarFlag(var, "export", d)
        if export:
@@ -419,7 +515,7 @@ def build_environment(d):
 def prunedir(topdir):
    # Delete everything reachable from the directory named in 'topdir'.
    # CAUTION:  This is dangerous!
-    for root, dirs, files in os.walk(topdir, topdown=False):
+    for root, dirs, files in os.walk(topdir, topdown = False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
@@ -434,7 +530,7 @@ def prunedir(topdir):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    # See if var ends with any of the suffixes listed and 
+    # See if var ends with any of the suffixes listed and
    # remove it if found
    for suffix in suffixes:
        if var.endswith(suffix):
@@ -446,169 +542,167 @@ def mkdirhier(dir):
    directory already exists like os.makedirs
    """
-    bb.debug(3, "mkdirhier(%s)" % dir)
+    bb.msg.debug(3, bb.msg.domain.Util, "mkdirhier(%s)" % dir)
    try:
        os.makedirs(dir)
-        bb.debug(2, "created " + dir)
+        bb.msg.debug(2, bb.msg.domain.Util, "created " + dir)
-    except OSError, e:
+    except OSError as e:
-        if e.errno != 17: raise e
+        if e.errno != errno.EEXIST:
+            raise e
-import stat
-def movefile(src,dest,newmtime=None,sstat=None):
+def movefile(src, dest, newmtime = None, sstat = None):
    """Moves a file from src to dest, preserving all permissions and
    attributes; mtime will be preserved even when moving across
    filesystems.  Returns true on success and false on failure. Move is
    atomic.
    """
-    #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")"
+    #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
    try:
        if not sstat:
-            sstat=os.lstat(src)
+            sstat = os.lstat(src)
-    except Exception, e:
+    except Exception as e:
-        print "movefile: Stating source file failed...", e
+        print("movefile: Stating source file failed...", e)
        return None
-    destexists=1
+    destexists = 1
    try:
-        dstat=os.lstat(dest)
+        dstat = os.lstat(dest)
    except:
-        dstat=os.lstat(os.path.dirname(dest))
+        dstat = os.lstat(os.path.dirname(dest))
-        destexists=0
+        destexists = 0
    if destexists:
        if stat.S_ISLNK(dstat[stat.ST_MODE]):
            try:
                os.unlink(dest)
-                destexists=0
+                destexists = 0
-            except Exception, e:
+            except Exception as e:
                pass
    if stat.S_ISLNK(sstat[stat.ST_MODE]):
        try:
-            target=os.readlink(src)
+            target = os.readlink(src)
            if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
                os.unlink(dest)
-            os.symlink(target,dest)
+            os.symlink(target, dest)
            #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
            os.unlink(src)
            return os.lstat(dest)
-        except Exception, e:
+        except Exception as e:
-            print "movefile: failed to properly create symlink:", dest, "->", target, e
+            print("movefile: failed to properly create symlink:", dest, "->", target, e)
            return None
-    renamefailed=1
+    renamefailed = 1
-    if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]:
+    if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
        try:
-            ret=os.rename(src,dest)
+            os.rename(src, dest)
-            renamefailed=0
+            renamefailed = 0
-        except Exception, e:
+        except Exception as e:
-            import errno
+            if e[0] != errno.EXDEV:
-            if e[0]!=errno.EXDEV:
                # Some random error.
-                print "movefile: Failed to move", src, "to", dest, e
+                print("movefile: Failed to move", src, "to", dest, e)
                return None
            # Invalid cross-device-link 'bind' mounted or actually Cross-Device
    if renamefailed:
-        didcopy=0
+        didcopy = 0
        if stat.S_ISREG(sstat[stat.ST_MODE]):
            try: # For safety copy then move it over.
-                shutil.copyfile(src,dest+"#new")
+                shutil.copyfile(src, dest + "#new")
-                os.rename(dest+"#new",dest)
+                os.rename(dest + "#new", dest)
-                didcopy=1
+                didcopy = 1
-            except Exception, e:
+            except Exception as e:
-                print 'movefile: copy', src, '->', dest, 'failed.', e
+                print('movefile: copy', src, '->', dest, 'failed.', e)
                return None
        else:
            #we don't yet handle special, so we need to fall back to /bin/mv
-            a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'")
+            a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
-            if a[0]!=0:
+            if a[0] != 0:
-                print "movefile: Failed to move special file:" + src + "' to '" + dest + "'", a
+                print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
                return None # failure
        try:
            if didcopy:
-                os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
+                os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
                os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
                os.unlink(src)
-        except Exception, e:
+        except Exception as e:
-            print "movefile: Failed to chown/chmod/unlink", dest, e
+            print("movefile: Failed to chown/chmod/unlink", dest, e)
            return None
    if newmtime:
-        os.utime(dest,(newmtime,newmtime))
+        os.utime(dest, (newmtime, newmtime))
    else:
        os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
-        newmtime=sstat[stat.ST_MTIME]
+        newmtime = sstat[stat.ST_MTIME]
    return newmtime
-def copyfile(src,dest,newmtime=None,sstat=None):
+def copyfile(src, dest, newmtime = None, sstat = None):
    """
    Copies a file from src to dest, preserving all permissions and
    attributes; mtime will be preserved even when moving across
    filesystems.  Returns true on success and false on failure.
    """
-    #print "copyfile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")"
+    #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
    try:
        if not sstat:
-            sstat=os.lstat(src)
+            sstat = os.lstat(src)
-    except Exception, e:
+    except Exception as e:
-        print "copyfile: Stating source file failed...", e
+        print("copyfile: Stating source file failed...", e)
        return False
-    destexists=1
+    destexists = 1
    try:
-        dstat=os.lstat(dest)
+        dstat = os.lstat(dest)
    except:
-        dstat=os.lstat(os.path.dirname(dest))
+        dstat = os.lstat(os.path.dirname(dest))
-        destexists=0
+        destexists = 0
    if destexists:
        if stat.S_ISLNK(dstat[stat.ST_MODE]):
            try:
                os.unlink(dest)
-                destexists=0
+                destexists = 0
-            except Exception, e:
+            except Exception as e:
                pass
    if stat.S_ISLNK(sstat[stat.ST_MODE]):
        try:
-            target=os.readlink(src)
+            target = os.readlink(src)
            if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
                os.unlink(dest)
-            os.symlink(target,dest)
+            os.symlink(target, dest)
            #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
            return os.lstat(dest)
-        except Exception, e:
+        except Exception as e:
-            print "copyfile: failed to properly create symlink:", dest, "->", target, e
+            print("copyfile: failed to properly create symlink:", dest, "->", target, e)
            return False
    if stat.S_ISREG(sstat[stat.ST_MODE]):
-            try: # For safety copy then move it over.
+        try: # For safety copy then move it over.
-                shutil.copyfile(src,dest+"#new")
+            shutil.copyfile(src, dest + "#new")
-                os.rename(dest+"#new",dest)
+            os.rename(dest + "#new", dest)
-            except Exception, e:
+        except Exception as e:
-                print 'copyfile: copy', src, '->', dest, 'failed.', e
+            print('copyfile: copy', src, '->', dest, 'failed.', e)
-                return False
+            return False
    else:
-            #we don't yet handle special, so we need to fall back to /bin/mv
+        #we don't yet handle special, so we need to fall back to /bin/mv
-            a=getstatusoutput("/bin/cp -f "+"'"+src+"' '"+dest+"'")
+        a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'")
-            if a[0]!=0:
+        if a[0] != 0:
-                print "copyfile: Failed to copy special file:" + src + "' to '" + dest + "'", a
+            print("copyfile: Failed to copy special file:" + src + "' to '" + dest + "'", a)
-                return False # failure
+            return False # failure
    try:
-        os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
+        os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
        os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
-    except Exception, e:
+    except Exception as e:
-        print "copyfile: Failed to chown/chmod/unlink", dest, e
+        print("copyfile: Failed to chown/chmod/unlink", dest, e)
        return False
    if newmtime:
-        os.utime(dest,(newmtime,newmtime))
+        os.utime(dest, (newmtime, newmtime))
    else:
        os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
-        newmtime=sstat[stat.ST_MTIME]
+        newmtime = sstat[stat.ST_MTIME]
    return newmtime
 def which(path, item, direction = 0):