85 files changed, 9871 insertions, 3023 deletions

diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py
index 23c22b65ef..76bc08a3ea 100644
--- a/bitbake/lib/bb/COW.py
+++ b/bitbake/lib/bb/COW.py
@@ -3,6 +3,8 @@
 #
 # Copyright (C) 2006 Tim Ansell
 #
+# SPDX-License-Identifier: GPL-2.0-only
+#
 # Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index 84a9051c13..15013540c2 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -9,12 +9,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.49.2"
+__version__ = "2.9.0"
 
 import sys
-if sys.version_info < (3, 5, 0):
-    raise RuntimeError("Sorry, python 3.5.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 8, 0):
+    raise RuntimeError("Sorry, python 3.8.0 or later is required for this version of bitbake")
 
+if sys.version_info < (3, 10, 0):
+    # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work"
+    # https://stackoverflow.com/questions/64797838/libgcc-s-so-1-must-be-installed-for-pthread-cancel-to-work
+    # https://bugs.ams1.psf.io/issue42888
+    # so ensure libgcc_s is loaded early on
+    import ctypes
+    libgcc_s = ctypes.CDLL('libgcc_s.so.1')
 
 class BBHandledException(Exception):
     """
@@ -58,8 +65,12 @@ class BBLoggerMixin(object):
         if not bb.event.worker_pid:
             if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if loglevel > bb.msg.loggerDefaultLogLevel:
+            if loglevel < bb.msg.loggerDefaultLogLevel:
                 return
+
+        if not isinstance(level, int) or not isinstance(msg, str):
+            mainlogger.warning("Invalid arguments in bbdebug: %s" % repr((level, msg,) + args))
+
         return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
@@ -71,6 +82,13 @@ class BBLoggerMixin(object):
     def verbnote(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 2, msg, *args, **kwargs)
 
+    def warnonce(self, msg, *args, **kwargs):
+        return self.log(logging.WARNING - 1, msg, *args, **kwargs)
+
+    def erroronce(self, msg, *args, **kwargs):
+        return self.log(logging.ERROR - 1, msg, *args, **kwargs)
+
+
 Logger = logging.getLoggerClass()
 class BBLogger(Logger, BBLoggerMixin):
     def __init__(self, name, *args, **kwargs):
@@ -157,9 +175,15 @@ def verbnote(*args):
 def warn(*args):
     mainlogger.warning(''.join(args))
 
+def warnonce(*args):
+    mainlogger.warnonce(''.join(args))
+
 def error(*args, **kwargs):
     mainlogger.error(''.join(args), extra=kwargs)
 
+def erroronce(*args):
+    mainlogger.erroronce(''.join(args))
+
 def fatal(*args, **kwargs):
     mainlogger.critical(''.join(args), extra=kwargs)
     raise BBHandledException()
diff --git a/bitbake/lib/bb/acl.py b/bitbake/lib/bb/acl.py
new file mode 100755
index 0000000000..0f41b275cf
--- /dev/null
+++ b/bitbake/lib/bb/acl.py
@@ -0,0 +1,215 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+
+import sys
+import ctypes
+import os
+import errno
+import pwd
+import grp
+
+libacl = ctypes.CDLL("libacl.so.1", use_errno=True)
+
+
+ACL_TYPE_ACCESS = 0x8000
+ACL_TYPE_DEFAULT = 0x4000
+
+ACL_FIRST_ENTRY = 0
+ACL_NEXT_ENTRY = 1
+
+ACL_UNDEFINED_TAG = 0x00
+ACL_USER_OBJ = 0x01
+ACL_USER = 0x02
+ACL_GROUP_OBJ = 0x04
+ACL_GROUP = 0x08
+ACL_MASK = 0x10
+ACL_OTHER = 0x20
+
+ACL_READ = 0x04
+ACL_WRITE = 0x02
+ACL_EXECUTE = 0x01
+
+acl_t = ctypes.c_void_p
+acl_entry_t = ctypes.c_void_p
+acl_permset_t = ctypes.c_void_p
+acl_perm_t = ctypes.c_uint
+
+acl_tag_t = ctypes.c_int
+
+libacl.acl_free.argtypes = [acl_t]
+
+
+def acl_free(acl):
+    libacl.acl_free(acl)
+
+
+libacl.acl_get_file.restype = acl_t
+libacl.acl_get_file.argtypes = [ctypes.c_char_p, ctypes.c_uint]
+
+
+def acl_get_file(path, typ):
+    acl = libacl.acl_get_file(os.fsencode(path), typ)
+    if acl is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err), str(path))
+
+    return acl
+
+
+libacl.acl_get_entry.argtypes = [acl_t, ctypes.c_int, ctypes.c_void_p]
+
+
+def acl_get_entry(acl, entry_id):
+    entry = acl_entry_t()
+    ret = libacl.acl_get_entry(acl, entry_id, ctypes.byref(entry))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    if ret == 0:
+        return None
+
+    return entry
+
+
+libacl.acl_get_tag_type.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_tag_type(entry_d):
+    tag = acl_tag_t()
+    ret = libacl.acl_get_tag_type(entry_d, ctypes.byref(tag))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return tag.value
+
+
+libacl.acl_get_qualifier.restype = ctypes.c_void_p
+libacl.acl_get_qualifier.argtypes = [acl_entry_t]
+
+
+def acl_get_qualifier(entry_d):
+    ret = libacl.acl_get_qualifier(entry_d)
+    if ret is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return ctypes.c_void_p(ret)
+
+
+libacl.acl_get_permset.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_permset(entry_d):
+    permset = acl_permset_t()
+    ret = libacl.acl_get_permset(entry_d, ctypes.byref(permset))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    return permset
+
+
+libacl.acl_get_perm.argtypes = [acl_permset_t, acl_perm_t]
+
+
+def acl_get_perm(permset_d, perm):
+    ret = libacl.acl_get_perm(permset_d, perm)
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return bool(ret)
+
+
+class Entry(object):
+    def __init__(self, tag, qualifier, mode):
+        self.tag = tag
+        self.qualifier = qualifier
+        self.mode = mode
+
+    def __str__(self):
+        typ = ""
+        qual = ""
+        if self.tag == ACL_USER:
+            typ = "user"
+            qual = pwd.getpwuid(self.qualifier).pw_name
+        elif self.tag == ACL_GROUP:
+            typ = "group"
+            qual = grp.getgrgid(self.qualifier).gr_name
+        elif self.tag == ACL_USER_OBJ:
+            typ = "user"
+        elif self.tag == ACL_GROUP_OBJ:
+            typ = "group"
+        elif self.tag == ACL_MASK:
+            typ = "mask"
+        elif self.tag == ACL_OTHER:
+            typ = "other"
+
+        r = "r" if self.mode & ACL_READ else "-"
+        w = "w" if self.mode & ACL_WRITE else "-"
+        x = "x" if self.mode & ACL_EXECUTE else "-"
+
+        return f"{typ}:{qual}:{r}{w}{x}"
+
+
+class ACL(object):
+    def __init__(self, acl):
+        self.acl = acl
+
+    def __del__(self):
+        acl_free(self.acl)
+
+    def entries(self):
+        entry_id = ACL_FIRST_ENTRY
+        while True:
+            entry = acl_get_entry(self.acl, entry_id)
+            if entry is None:
+                break
+
+            permset = acl_get_permset(entry)
+
+            mode = 0
+            for m in (ACL_READ, ACL_WRITE, ACL_EXECUTE):
+                if acl_get_perm(permset, m):
+                    mode |= m
+
+            qualifier = None
+            tag = acl_get_tag_type(entry)
+
+            if tag == ACL_USER or tag == ACL_GROUP:
+                qual = acl_get_qualifier(entry)
+                qualifier = ctypes.cast(qual, ctypes.POINTER(ctypes.c_int))[0]
+
+            yield Entry(tag, qualifier, mode)
+
+            entry_id = ACL_NEXT_ENTRY
+
+    @classmethod
+    def from_path(cls, path, typ):
+        acl = acl_get_file(path, typ)
+        return cls(acl)
+
+
+def main():
+    import argparse
+    import pwd
+    import grp
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    acl = ACL.from_path(args.path, ACL_TYPE_ACCESS)
+    for entry in acl.entries():
+        print(str(entry))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/bitbake/lib/bb/asyncrpc/__init__.py b/bitbake/lib/bb/asyncrpc/__init__.py
new file mode 100644
index 0000000000..639e1607f8
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+from .client import AsyncClient, Client, ClientPool
+from .serv import AsyncServer, AsyncServerConnection
+from .connection import DEFAULT_MAX_CHUNK
+from .exceptions import (
+    ClientError,
+    ServerError,
+    ConnectionClosedError,
+    InvokeError,
+)
diff --git a/bitbake/lib/bb/asyncrpc/client.py b/bitbake/lib/bb/asyncrpc/client.py
new file mode 100644
index 0000000000..a350b4fb12
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/client.py
@@ -0,0 +1,313 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import socket
+import sys
+import re
+import contextlib
+from threading import Thread
+from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK
+from .exceptions import ConnectionClosedError, InvokeError
+
+UNIX_PREFIX = "unix://"
+WS_PREFIX = "ws://"
+WSS_PREFIX = "wss://"
+
+ADDR_TYPE_UNIX = 0
+ADDR_TYPE_TCP = 1
+ADDR_TYPE_WS = 2
+
+def parse_address(addr):
+    if addr.startswith(UNIX_PREFIX):
+        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
+    elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
+        return (ADDR_TYPE_WS, (addr,))
+    else:
+        m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
+        if m is not None:
+            host = m.group("host")
+            port = m.group("port")
+        else:
+            host, port = addr.split(":")
+
+        return (ADDR_TYPE_TCP, (host, int(port)))
+
+class AsyncClient(object):
+    def __init__(
+        self,
+        proto_name,
+        proto_version,
+        logger,
+        timeout=30,
+        server_headers=False,
+        headers={},
+    ):
+        self.socket = None
+        self.max_chunk = DEFAULT_MAX_CHUNK
+        self.proto_name = proto_name
+        self.proto_version = proto_version
+        self.logger = logger
+        self.timeout = timeout
+        self.needs_server_headers = server_headers
+        self.server_headers = {}
+        self.headers = headers
+
+    async def connect_tcp(self, address, port):
+        async def connect_sock():
+            reader, writer = await asyncio.open_connection(address, port)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
+
+        self._connect_sock = connect_sock
+
+    async def connect_unix(self, path):
+        async def connect_sock():
+            # AF_UNIX has path length issues so chdir here to workaround
+            cwd = os.getcwd()
+            try:
+                os.chdir(os.path.dirname(path))
+                # The socket must be opened synchronously so that CWD doesn't get
+                # changed out from underneath us so we pass as a sock into asyncio
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+                sock.connect(os.path.basename(path))
+            finally:
+                os.chdir(cwd)
+            reader, writer = await asyncio.open_unix_connection(sock=sock)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
+
+        self._connect_sock = connect_sock
+
+    async def connect_websocket(self, uri):
+        import websockets
+
+        async def connect_sock():
+            websocket = await websockets.connect(uri, ping_interval=None)
+            return WebsocketConnection(websocket, self.timeout)
+
+        self._connect_sock = connect_sock
+
+    async def setup_connection(self):
+        # Send headers
+        await self.socket.send("%s %s" % (self.proto_name, self.proto_version))
+        await self.socket.send(
+            "needs-headers: %s" % ("true" if self.needs_server_headers else "false")
+        )
+        for k, v in self.headers.items():
+            await self.socket.send("%s: %s" % (k, v))
+
+        # End of headers
+        await self.socket.send("")
+
+        self.server_headers = {}
+        if self.needs_server_headers:
+            while True:
+                line = await self.socket.recv()
+                if not line:
+                    # End headers
+                    break
+                tag, value = line.split(":", 1)
+                self.server_headers[tag.lower()] = value.strip()
+
+    async def get_header(self, tag, default):
+        await self.connect()
+        return self.server_headers.get(tag, default)
+
+    async def connect(self):
+        if self.socket is None:
+            self.socket = await self._connect_sock()
+            await self.setup_connection()
+
+    async def disconnect(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None
+
+    async def close(self):
+        await self.disconnect()
+
+    async def _send_wrapper(self, proc):
+        count = 0
+        while True:
+            try:
+                await self.connect()
+                return await proc()
+            except (
+                OSError,
+                ConnectionError,
+                ConnectionClosedError,
+                json.JSONDecodeError,
+                UnicodeDecodeError,
+            ) as e:
+                self.logger.warning("Error talking to server: %s" % e)
+                if count >= 3:
+                    if not isinstance(e, ConnectionError):
+                        raise ConnectionError(str(e))
+                    raise e
+                await self.close()
+                count += 1
+
+    def check_invoke_error(self, msg):
+        if isinstance(msg, dict) and "invoke-error" in msg:
+            raise InvokeError(msg["invoke-error"]["message"])
+
+    async def invoke(self, msg):
+        async def proc():
+            await self.socket.send_message(msg)
+            return await self.socket.recv_message()
+
+        result = await self._send_wrapper(proc)
+        self.check_invoke_error(result)
+        return result
+
+    async def ping(self):
+        return await self.invoke({"ping": {}})
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+
+class Client(object):
+    def __init__(self):
+        self.client = self._get_async_client()
+        self.loop = asyncio.new_event_loop()
+
+        # Override any pre-existing loop.
+        # Without this, the PR server export selftest triggers a hang
+        # when running with Python 3.7.  The drawback is that there is
+        # potential for issues if the PR and hash equiv (or some new)
+        # clients need to both be instantiated in the same process.
+        # This should be revisited if/when Python 3.9 becomes the
+        # minimum required version for BitBake, as it seems not
+        # required (but harmless) with it.
+        asyncio.set_event_loop(self.loop)
+
+        self._add_methods("connect_tcp", "ping")
+
+    @abc.abstractmethod
+    def _get_async_client(self):
+        pass
+
+    def _get_downcall_wrapper(self, downcall):
+        def wrapper(*args, **kwargs):
+            return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+        return wrapper
+
+    def _add_methods(self, *methods):
+        for m in methods:
+            downcall = getattr(self.client, m)
+            setattr(self, m, self._get_downcall_wrapper(downcall))
+
+    def connect_unix(self, path):
+        self.loop.run_until_complete(self.client.connect_unix(path))
+        self.loop.run_until_complete(self.client.connect())
+
+    @property
+    def max_chunk(self):
+        return self.client.max_chunk
+
+    @max_chunk.setter
+    def max_chunk(self, value):
+        self.client.max_chunk = value
+
+    def disconnect(self):
+        self.loop.run_until_complete(self.client.close())
+
+    def close(self):
+        if self.loop:
+            self.loop.run_until_complete(self.client.close())
+            if sys.version_info >= (3, 6):
+                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+        self.loop = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False
+
+
+class ClientPool(object):
+    def __init__(self, max_clients):
+        self.avail_clients = []
+        self.num_clients = 0
+        self.max_clients = max_clients
+        self.loop = None
+        self.client_condition = None
+
+    @abc.abstractmethod
+    async def _new_client(self):
+        raise NotImplementedError("Must be implemented in derived class")
+
+    def close(self):
+        if self.client_condition:
+            self.client_condition = None
+
+        if self.loop:
+            self.loop.run_until_complete(self.__close_clients())
+            self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+            self.loop = None
+
+    def run_tasks(self, tasks):
+        if not self.loop:
+            self.loop = asyncio.new_event_loop()
+
+        thread = Thread(target=self.__thread_main, args=(tasks,))
+        thread.start()
+        thread.join()
+
+    @contextlib.asynccontextmanager
+    async def get_client(self):
+        async with self.client_condition:
+            if self.avail_clients:
+                client = self.avail_clients.pop()
+            elif self.num_clients < self.max_clients:
+                self.num_clients += 1
+                client = await self._new_client()
+            else:
+                while not self.avail_clients:
+                    await self.client_condition.wait()
+                client = self.avail_clients.pop()
+
+        try:
+            yield client
+        finally:
+            async with self.client_condition:
+                self.avail_clients.append(client)
+                self.client_condition.notify()
+
+    def __thread_main(self, tasks):
+        async def process_task(task):
+            async with self.get_client() as client:
+                await task(client)
+
+        asyncio.set_event_loop(self.loop)
+        if not self.client_condition:
+            self.client_condition = asyncio.Condition()
+        tasks = [process_task(t) for t in tasks]
+        self.loop.run_until_complete(asyncio.gather(*tasks))
+
+    async def __close_clients(self):
+        for c in self.avail_clients:
+            await c.close()
+        self.avail_clients = []
+        self.num_clients = 0
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False
diff --git a/bitbake/lib/bb/asyncrpc/connection.py b/bitbake/lib/bb/asyncrpc/connection.py
new file mode 100644
index 0000000000..7f0cf6ba96
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/connection.py
@@ -0,0 +1,146 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import asyncio
+import itertools
+import json
+from datetime import datetime
+from .exceptions import ClientError, ConnectionClosedError
+
+
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
+
+
+def chunkify(msg, max_chunk):
+    if len(msg) < max_chunk - 1:
+        yield "".join((msg, "\n"))
+    else:
+        yield "".join((json.dumps({"chunk-stream": None}), "\n"))
+
+        args = [iter(msg)] * (max_chunk - 1)
+        for m in map("".join, itertools.zip_longest(*args, fillvalue="")):
+            yield "".join(itertools.chain(m, "\n"))
+        yield "\n"
+
+
+def json_serialize(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializeable" % type(obj))
+
+
+class StreamConnection(object):
+    def __init__(self, reader, writer, timeout, max_chunk=DEFAULT_MAX_CHUNK):
+        self.reader = reader
+        self.writer = writer
+        self.timeout = timeout
+        self.max_chunk = max_chunk
+
+    @property
+    def address(self):
+        return self.writer.get_extra_info("peername")
+
+    async def send_message(self, msg):
+        for c in chunkify(json.dumps(msg, default=json_serialize), self.max_chunk):
+            self.writer.write(c.encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv_message(self):
+        l = await self.recv()
+
+        m = json.loads(l)
+        if not m:
+            return m
+
+        if "chunk-stream" in m:
+            lines = []
+            while True:
+                l = await self.recv()
+                if not l:
+                    break
+                lines.append(l)
+
+            m = json.loads("".join(lines))
+
+        return m
+
+    async def send(self, msg):
+        self.writer.write(("%s\n" % msg).encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv(self):
+        if self.timeout < 0:
+            line = await self.reader.readline()
+        else:
+            try:
+                line = await asyncio.wait_for(self.reader.readline(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+
+        if not line:
+            raise ConnectionClosedError("Connection closed")
+
+        line = line.decode("utf-8")
+
+        if not line.endswith("\n"):
+            raise ConnectionError("Bad message %r" % (line))
+
+        return line.rstrip()
+
+    async def close(self):
+        self.reader = None
+        if self.writer is not None:
+            self.writer.close()
+            self.writer = None
+
+
+class WebsocketConnection(object):
+    def __init__(self, socket, timeout):
+        self.socket = socket
+        self.timeout = timeout
+
+    @property
+    def address(self):
+        return ":".join(str(s) for s in self.socket.remote_address)
+
+    async def send_message(self, msg):
+        await self.send(json.dumps(msg, default=json_serialize))
+
+    async def recv_message(self):
+        m = await self.recv()
+        return json.loads(m)
+
+    async def send(self, msg):
+        import websockets.exceptions
+
+        try:
+            await self.socket.send(msg)
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def recv(self):
+        import websockets.exceptions
+
+        try:
+            if self.timeout < 0:
+                return await self.socket.recv()
+
+            try:
+                return await asyncio.wait_for(self.socket.recv(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def close(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None
diff --git a/bitbake/lib/bb/asyncrpc/exceptions.py b/bitbake/lib/bb/asyncrpc/exceptions.py
new file mode 100644
index 0000000000..ae1043a38b
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/exceptions.py
@@ -0,0 +1,21 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+class ClientError(Exception):
+    pass
+
+
+class InvokeError(Exception):
+    pass
+
+
+class ServerError(Exception):
+    pass
+
+
+class ConnectionClosedError(Exception):
+    pass
diff --git a/bitbake/lib/bb/asyncrpc/serv.py b/bitbake/lib/bb/asyncrpc/serv.py
new file mode 100644
index 0000000000..a66117acad
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/serv.py
@@ -0,0 +1,391 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import signal
+import socket
+import sys
+import multiprocessing
+import logging
+from .connection import StreamConnection, WebsocketConnection
+from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError
+
+
+class ClientLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return f"[Client {self.extra['address']}] {msg}", kwargs
+
+
+class AsyncServerConnection(object):
+    # If a handler returns this object (e.g. `return self.NO_RESPONSE`), no
+    # return message will be automatically be sent back to the client
+    NO_RESPONSE = object()
+
+    def __init__(self, socket, proto_name, logger):
+        self.socket = socket
+        self.proto_name = proto_name
+        self.handlers = {
+            "ping": self.handle_ping,
+        }
+        self.logger = ClientLoggerAdapter(
+            logger,
+            {
+                "address": socket.address,
+            },
+        )
+        self.client_headers = {}
+
+    async def close(self):
+        await self.socket.close()
+
+    async def handle_headers(self, headers):
+        return {}
+
+    async def process_requests(self):
+        try:
+            self.logger.info("Client %r connected" % (self.socket.address,))
+
+            # Read protocol and version
+            client_protocol = await self.socket.recv()
+            if not client_protocol:
+                return
+
+            (client_proto_name, client_proto_version) = client_protocol.split()
+            if client_proto_name != self.proto_name:
+                self.logger.debug("Rejecting invalid protocol %s" % (self.proto_name))
+                return
+
+            self.proto_version = tuple(int(v) for v in client_proto_version.split("."))
+            if not self.validate_proto_version():
+                self.logger.debug(
+                    "Rejecting invalid protocol version %s" % (client_proto_version)
+                )
+                return
+
+            # Read headers
+            self.client_headers = {}
+            while True:
+                header = await self.socket.recv()
+                if not header:
+                    # Empty line. End of headers
+                    break
+                tag, value = header.split(":", 1)
+                self.client_headers[tag.lower()] = value.strip()
+
+            if self.client_headers.get("needs-headers", "false") == "true":
+                for k, v in (await self.handle_headers(self.client_headers)).items():
+                    await self.socket.send("%s: %s" % (k, v))
+                await self.socket.send("")
+
+            # Handle messages
+            while True:
+                d = await self.socket.recv_message()
+                if d is None:
+                    break
+                try:
+                    response = await self.dispatch_message(d)
+                except InvokeError as e:
+                    await self.socket.send_message(
+                        {"invoke-error": {"message": str(e)}}
+                    )
+                    break
+
+                if response is not self.NO_RESPONSE:
+                    await self.socket.send_message(response)
+
+        except ConnectionClosedError as e:
+            self.logger.info(str(e))
+        except (ClientError, ConnectionError) as e:
+            self.logger.error(str(e))
+        finally:
+            await self.close()
+
+    async def dispatch_message(self, msg):
+        for k in self.handlers.keys():
+            if k in msg:
+                self.logger.debug("Handling %s" % k)
+                return await self.handlers[k](msg[k])
+
+        raise ClientError("Unrecognized command %r" % msg)
+
+    async def handle_ping(self, request):
+        return {"alive": True}
+
+
+class StreamServer(object):
+    def __init__(self, handler, logger):
+        self.handler = handler
+        self.logger = logger
+        self.closed = False
+
+    async def handle_stream_client(self, reader, writer):
+        # writer.transport.set_write_buffer_limits(0)
+        socket = StreamConnection(reader, writer, -1)
+        if self.closed:
+            await socket.close()
+            return
+
+        await self.handler(socket)
+
+    async def stop(self):
+        self.closed = True
+
+
+class TCPStreamServer(StreamServer):
+    def __init__(self, host, port, handler, logger):
+        super().__init__(handler, logger)
+        self.host = host
+        self.port = port
+
+    def start(self, loop):
+        self.server = loop.run_until_complete(
+            asyncio.start_server(self.handle_stream_client, self.host, self.port)
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+            # Newer python does this automatically. Do it manually here for
+            # maximum compatibility
+            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+
+class UnixStreamServer(StreamServer):
+    def __init__(self, path, handler, logger):
+        super().__init__(handler, logger)
+        self.path = path
+
+    def start(self, loop):
+        cwd = os.getcwd()
+        try:
+            # Work around path length limits in AF_UNIX
+            os.chdir(os.path.dirname(self.path))
+            self.server = loop.run_until_complete(
+                asyncio.start_unix_server(
+                    self.handle_stream_client, os.path.basename(self.path)
+                )
+            )
+        finally:
+            os.chdir(cwd)
+
+        self.logger.debug("Listening on %r" % self.path)
+        self.address = "unix://%s" % os.path.abspath(self.path)
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        os.unlink(self.path)
+
+
+class WebsocketsServer(object):
+    def __init__(self, host, port, handler, logger):
+        self.host = host
+        self.port = port
+        self.handler = handler
+        self.logger = logger
+
+    def start(self, loop):
+        import websockets.server
+
+        self.server = loop.run_until_complete(
+            websockets.server.serve(
+                self.client_handler,
+                self.host,
+                self.port,
+                ping_interval=None,
+            )
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "ws://[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "ws://%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+    async def client_handler(self, websocket):
+        socket = WebsocketConnection(websocket, -1)
+        await self.handler(socket)
+
+
+class AsyncServer(object):
+    def __init__(self, logger):
+        self.logger = logger
+        self.loop = None
+        self.run_tasks = []
+
+    def start_tcp_server(self, host, port):
+        self.server = TCPStreamServer(host, port, self._client_handler, self.logger)
+
+    def start_unix_server(self, path):
+        self.server = UnixStreamServer(path, self._client_handler, self.logger)
+
+    def start_websocket_server(self, host, port):
+        self.server = WebsocketsServer(host, port, self._client_handler, self.logger)
+
+    async def _client_handler(self, socket):
+        address = socket.address
+        try:
+            client = self.accept_client(socket)
+            await client.process_requests()
+        except Exception as e:
+            import traceback
+
+            self.logger.error(
+                "Error from client %s: %s" % (address, str(e)), exc_info=True
+            )
+            traceback.print_exc()
+        finally:
+            self.logger.debug("Client %s disconnected", address)
+            await socket.close()
+
+    @abc.abstractmethod
+    def accept_client(self, socket):
+        pass
+
+    async def stop(self):
+        self.logger.debug("Stopping server")
+        await self.server.stop()
+
+    def start(self):
+        tasks = self.server.start(self.loop)
+        self.address = self.server.address
+        return tasks
+
+    def signal_handler(self):
+        self.logger.debug("Got exit signal")
+        self.loop.create_task(self.stop())
+
+    def _serve_forever(self, tasks):
+        try:
+            self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGINT, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGQUIT, self.signal_handler)
+            signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM])
+
+            self.loop.run_until_complete(asyncio.gather(*tasks))
+
+            self.logger.debug("Server shutting down")
+        finally:
+            self.server.cleanup()
+
+    def serve_forever(self):
+        """
+        Serve requests in the current process
+        """
+        self._create_loop()
+        tasks = self.start()
+        self._serve_forever(tasks)
+        self.loop.close()
+
+    def _create_loop(self):
+        # Create loop and override any loop that may have existed in
+        # a parent process.  It is possible that the usecases of
+        # serve_forever might be constrained enough to allow using
+        # get_event_loop here, but better safe than sorry for now.
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self.loop)
+
+    def serve_as_process(self, *, prefunc=None, args=(), log_level=None):
+        """
+        Serve requests in a child process
+        """
+
+        def run(queue):
+            # Create loop and override any loop that may have existed
+            # in a parent process.  Without doing this and instead
+            # using get_event_loop, at the very minimum the hashserv
+            # unit tests will hang when running the second test.
+            # This happens since get_event_loop in the spawned server
+            # process for the second testcase ends up with the loop
+            # from the hashserv client created in the unit test process
+            # when running the first testcase.  The problem is somewhat
+            # more general, though, as any potential use of asyncio in
+            # Cooker could create a loop that needs to replaced in this
+            # new process.
+            self._create_loop()
+            try:
+                self.address = None
+                tasks = self.start()
+            finally:
+                # Always put the server address to wake up the parent task
+                queue.put(self.address)
+                queue.close()
+
+            if prefunc is not None:
+                prefunc(self, *args)
+
+            if log_level is not None:
+                self.logger.setLevel(log_level)
+
+            self._serve_forever(tasks)
+
+            if sys.version_info >= (3, 6):
+                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+
+        queue = multiprocessing.Queue()
+
+        # Temporarily block SIGTERM. The server process will inherit this
+        # block which will ensure it doesn't receive the SIGTERM until the
+        # handler is ready for it
+        mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTERM])
+        try:
+            self.process = multiprocessing.Process(target=run, args=(queue,))
+            self.process.start()
+
+            self.address = queue.get()
+            queue.close()
+            queue.join_thread()
+
+            return self.process
+        finally:
+            signal.pthread_sigmask(signal.SIG_SETMASK, mask)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index f4f897e41a..44d08f5c55 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -20,10 +20,12 @@ import itertools
 import time
 import re
 import stat
+import datetime
 import bb
 import bb.msg
 import bb.process
 import bb.progress
+from io import StringIO
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')
@@ -176,7 +178,9 @@ class StdoutNoopContextManager:
 
     @property
     def name(self):
-        return sys.stdout.name
+        if "name" in dir(sys.stdout):
+            return sys.stdout.name
+        return "<mem>"
 
 
 def exec_func(func, d, dirs = None):
@@ -295,9 +299,25 @@ def exec_func_python(func, d, runfile, cwd=None):
         lineno = int(d.getVarFlag(func, "lineno", False))
         bb.methodpool.insert_method(func, text, fn, lineno - 1)
 
-        comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
-        utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
+        if verboseStdoutLogging:
+            sys.stdout.flush()
+            sys.stderr.flush()
+            currout = sys.stdout
+            currerr = sys.stderr
+            sys.stderr = sys.stdout = execio = StringIO()
+        comp = utils.better_compile(code, func, "exec_func_python() autogenerated")
+        utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated")
     finally:
+        if verboseStdoutLogging:
+            execio.flush()
+            logger.plain("%s" % execio.getvalue())
+            sys.stdout = currout
+            sys.stderr = currerr
+            execio.close()
+        # We want any stdout/stderr to be printed before any other log messages to make debugging
+        # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this.
+        sys.stdout.flush()
+        sys.stderr.flush()
         bb.debug(2, "Python function %s finished" % func)
 
         if cwd and olddir:
@@ -436,7 +456,11 @@ exit $ret
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
-    if verboseStdoutLogging:
+    # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either
+    # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively",
+    # ie. inside another function, in which case stdout is already being captured so we don't
+    # want to Tee here as output would be printed twice, and out of order.
+    if verboseStdoutLogging and sys.stdout == sys.__stdout__:
         logfile = LogTee(logger, StdoutNoopContextManager())
     else:
         logfile = StdoutNoopContextManager()
@@ -565,10 +589,8 @@ exit $ret
 def _task_data(fn, task, d):
     localdata = bb.data.createCopy(d)
     localdata.setVar('BB_FILENAME', fn)
-    localdata.setVar('BB_CURRENTTASK', task[3:])
     localdata.setVar('OVERRIDES', 'task-%s:%s' %
                      (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
-    localdata.finalize()
     bb.data.expandKeys(localdata)
     return localdata
 
@@ -579,7 +601,7 @@ def _exec_task(fn, task, d, quieterr):
     running it with its own local metadata, and with some useful variables set.
     """
     if not d.getVarFlag(task, 'task', False):
-        event.fire(TaskInvalid(task, d), d)
+        event.fire(TaskInvalid(task, fn, d), d)
         logger.error("No such task: %s" % task)
         return 1
 
@@ -615,7 +637,8 @@ def _exec_task(fn, task, d, quieterr):
     logorder = os.path.join(tempdir, 'log.task_order')
     try:
         with open(logorder, 'a') as logorderfile:
-            logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
+            timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f")
+            logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase))
     except OSError:
         logger.exception("Opening log file '%s'", logorder)
         pass
@@ -682,47 +705,55 @@ def _exec_task(fn, task, d, quieterr):
     try:
         try:
             event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
-        except (bb.BBHandledException, SystemExit):
-            return 1
 
-        try:
             for func in (prefuncs or '').split():
                 exec_func(func, localdata)
             exec_func(task, localdata)
             for func in (postfuncs or '').split():
                 exec_func(func, localdata)
-        except bb.BBHandledException:
-            event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
-            return 1
-        except Exception as exc:
-            if quieterr:
-                event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
-            else:
-                errprinted = errchk.triggered
-                logger.error(str(exc))
-                event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
-            return 1
-    finally:
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-        bblogger.removeHandler(handler)
-
-        # Restore the backup fds
-        os.dup2(osi[0], osi[1])
-        os.dup2(oso[0], oso[1])
-        os.dup2(ose[0], ose[1])
-
-        # Close the backup fds
-        os.close(osi[0])
-        os.close(oso[0])
-        os.close(ose[0])
+        finally:
+            # Need to flush and close the logs before sending events where the
+            # UI may try to look at the logs.
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            bblogger.removeHandler(handler)
+
+            # Restore the backup fds
+            os.dup2(osi[0], osi[1])
+            os.dup2(oso[0], oso[1])
+            os.dup2(ose[0], ose[1])
+
+            # Close the backup fds
+            os.close(osi[0])
+            os.close(oso[0])
+            os.close(ose[0])
+
+            logfile.close()
+            if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
+                logger.debug2("Zero size logfn %s, removing", logfn)
+                bb.utils.remove(logfn)
+                bb.utils.remove(loglink)
+    except (Exception, SystemExit) as exc:
+        handled = False
+        if isinstance(exc, bb.BBHandledException):
+            handled = True
+
+        if quieterr:
+            if not handled:
+                logger.warning(repr(exc))
+            event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
+        else:
+            errprinted = errchk.triggered
+            # If the output is already on stdout, we've printed the information in the
+            # logs once already so don't duplicate
+            if verboseStdoutLogging or handled:
+                errprinted = True
+            if not handled:
+                logger.error(repr(exc))
+            event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
+        return 1
 
-        logfile.close()
-        if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
-            logger.debug2("Zero size logfn %s, removing", logfn)
-            bb.utils.remove(logfn)
-            bb.utils.remove(loglink)
     event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
@@ -760,132 +791,92 @@ def exec_task(fn, task, d, profile = False):
             event.fire(failedevent, d)
         return 1
 
-def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
+def _get_cleanmask(taskname, mcfn):
     """
-    Internal stamp helper function
-    Makes sure the stamp directory exists
+    Internal stamp helper function to generate stamp cleaning mask
     Returns the stamp path+filename
 
     In the bitbake core, d can be a CacheData and file_name will be set.
     When called in task context, d will be a data store, file_name will not be set
     """
-    taskflagname = taskname
-    if taskname.endswith("_setscene") and taskname != "do_setscene":
-        taskflagname = taskname.replace("_setscene", "")
-
-    if file_name:
-        stamp = d.stamp[file_name]
-        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
-    else:
-        stamp = d.getVar('STAMP')
-        file_name = d.getVar('BB_FILENAME')
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+    cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn)
+    taskflagname = taskname.replace("_setscene", "")
+    if cleanmask:
+        return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
+    return []
+
+def clean_stamp_mcfn(task, mcfn):
+    cleanmask = _get_cleanmask(task, mcfn)
+    for mask in cleanmask:
+        for name in glob.glob(mask):
+            # Preserve sigdata files in the stamps directory
+            if "sigdata" in name or "sigbasedata" in name:
+                continue
+            # Preserve taint files in the stamps directory
+            if name.endswith('.taint'):
+                continue
+            os.unlink(name)
 
-    if baseonly:
-        return stamp
-    if noextra:
-        extrainfo = ""
+def clean_stamp(task, d):
+    mcfn = d.getVar('BB_FILENAME')
+    clean_stamp_mcfn(task, mcfn)
 
-    if not stamp:
-        return
+def make_stamp_mcfn(task, mcfn):
 
-    stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
+    basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn)
 
-    stampdir = os.path.dirname(stamp)
+    stampdir = os.path.dirname(basestamp)
     if cached_mtime_noerror(stampdir) == 0:
         bb.utils.mkdirhier(stampdir)
 
-    return stamp
+    clean_stamp_mcfn(task, mcfn)
 
-def stamp_cleanmask_internal(taskname, d, file_name):
-    """
-    Internal stamp helper function to generate stamp cleaning mask
-    Returns the stamp path+filename
+    # Remove the file and recreate to force timestamp
+    # change on broken NFS filesystems
+    if basestamp:
+        bb.utils.remove(basestamp)
+        open(basestamp, "w").close()
 
-    In the bitbake core, d can be a CacheData and file_name will be set.
-    When called in task context, d will be a data store, file_name will not be set
+def make_stamp(task, d):
     """
-    taskflagname = taskname
-    if taskname.endswith("_setscene") and taskname != "do_setscene":
-        taskflagname = taskname.replace("_setscene", "")
-
-    if file_name:
-        stamp = d.stampclean[file_name]
-        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
-    else:
-        stamp = d.getVar('STAMPCLEAN')
-        file_name = d.getVar('BB_FILENAME')
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+    Creates/updates a stamp for a given task
+    """
+    mcfn = d.getVar('BB_FILENAME')
 
-    if not stamp:
-        return []
+    make_stamp_mcfn(task, mcfn)
 
-    cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
+    # If we're in task context, write out a signature file for each task
+    # as it completes
+    if not task.endswith("_setscene"):
+        stampbase = bb.parse.siggen.stampfile_base(mcfn)
+        bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True)
 
-    return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
 
-def make_stamp(task, d, file_name = None):
-    """
-    Creates/updates a stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    cleanmask = stamp_cleanmask_internal(task, d, file_name)
+def find_stale_stamps(task, mcfn):
+    current = bb.parse.siggen.stampfile_mcfn(task, mcfn)
+    current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn)
+    cleanmask = _get_cleanmask(task, mcfn)
+    found = []
     for mask in cleanmask:
         for name in glob.glob(mask):
-            # Preserve sigdata files in the stamps directory
             if "sigdata" in name or "sigbasedata" in name:
                 continue
-            # Preserve taint files in the stamps directory
             if name.endswith('.taint'):
                 continue
-            os.unlink(name)
-
-    stamp = stamp_internal(task, d, file_name)
-    # Remove the file and recreate to force timestamp
-    # change on broken NFS filesystems
-    if stamp:
-        bb.utils.remove(stamp)
-        open(stamp, "w").close()
-
-    # If we're in task context, write out a signature file for each task
-    # as it completes
-    if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
-        stampbase = stamp_internal(task, d, None, True)
-        file_name = d.getVar('BB_FILENAME')
-        bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
-
-def del_stamp(task, d, file_name = None):
-    """
-    Removes a stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    stamp = stamp_internal(task, d, file_name)
-    bb.utils.remove(stamp)
+            if name == current or name == current2:
+                continue
+            logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2))
+            found.append(name)
+    return found
 
-def write_taint(task, d, file_name = None):
+def write_taint(task, d):
     """
     Creates a "taint" file which will force the specified task and its
     dependents to be re-run the next time by influencing the value of its
     taskhash.
-    (d can be a data dict or dataCache)
     """
-    import uuid
-    if file_name:
-        taintfn = d.stamp[file_name] + '.' + task + '.taint'
-    else:
-        taintfn = d.getVar('STAMP') + '.' + task + '.taint'
-    bb.utils.mkdirhier(os.path.dirname(taintfn))
-    # The specific content of the taint file is not really important,
-    # we just need it to be random, so a random UUID is used
-    with open(taintfn, 'w') as taintf:
-        taintf.write(str(uuid.uuid4()))
-
-def stampfile(taskname, d, file_name = None, noextra=False):
-    """
-    Return the stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    return stamp_internal(taskname, d, file_name, noextra=noextra)
+    mcfn = d.getVar('BB_FILENAME')
+    bb.parse.siggen.invalidate_task(task, mcfn)
 
 def add_tasks(tasklist, d):
     task_deps = d.getVar('_task_deps', False)
@@ -910,6 +901,11 @@ def add_tasks(tasklist, d):
                 task_deps[name] = {}
             if name in flags:
                 deptask = d.expand(flags[name])
+                if name in ['noexec', 'fakeroot', 'nostamp']:
+                    if deptask != '1':
+                        bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' "
+                                "will result in the flag not being set. See YP bug #13808.".format(name))
+
                 task_deps[name][task] = deptask
         getTask('mcdepends')
         getTask('depends')
@@ -1008,6 +1004,8 @@ def tasksbetween(task_start, task_end, d):
     def follow_chain(task, endtask, chain=None):
         if not chain:
             chain = []
+        if task in chain:
+            bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain)))
         chain.append(task)
         for othertask in tasks:
             if othertask == task:
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index aea2b8bc11..18d5574a31 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -19,14 +19,16 @@
 import os
 import logging
 import pickle
-from collections import defaultdict, Mapping
+from collections import defaultdict
+from collections.abc import Mapping
 import bb.utils
 from bb import PrefixLoggerAdapter
 import re
+import shutil
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "154"
+__cache_version__ = "155"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -53,12 +55,12 @@ class RecipeInfoCommon(object):
 
     @classmethod
     def pkgvar(cls, var, packages, metadata):
-        return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata))
+        return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata))
                     for pkg in packages)
 
     @classmethod
     def taskvar(cls, var, tasks, metadata):
-        return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata))
+        return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata))
                     for task in tasks)
 
     @classmethod
@@ -103,7 +105,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         self.tasks = metadata.getVar('__BBTASKS', False)
 
-        self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
+        self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {}
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
 
         self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
@@ -126,6 +128,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.inherits         = self.getvar('__inherit_cache', metadata, expand=False)
         self.fakerootenv      = self.getvar('FAKEROOTENV', metadata)
         self.fakerootdirs     = self.getvar('FAKEROOTDIRS', metadata)
+        self.fakerootlogs     = self.getvar('FAKEROOTLOGS', metadata)
         self.fakerootnoenv    = self.getvar('FAKEROOTNOENV', metadata)
         self.extradepsfunc    = self.getvar('calculate_extra_depends', metadata)
 
@@ -163,6 +166,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv = {}
         cachedata.fakerootnoenv = {}
         cachedata.fakerootdirs = {}
+        cachedata.fakerootlogs = {}
         cachedata.extradepsfunc = {}
 
     def add_cacheData(self, cachedata, fn):
@@ -212,7 +216,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         # Collect files we may need for possible world-dep
         # calculations
-        if not self.not_world:
+        if not bb.utils.to_boolean(self.not_world):
             cachedata.possible_world.append(fn)
         #else:
         #    logger.debug2("EXCLUDE FROM WORLD: %s", fn)
@@ -231,17 +235,116 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv[fn] = self.fakerootenv
         cachedata.fakerootnoenv[fn] = self.fakerootnoenv
         cachedata.fakerootdirs[fn] = self.fakerootdirs
+        cachedata.fakerootlogs[fn] = self.fakerootlogs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
+
+class SiggenRecipeInfo(RecipeInfoCommon):
+    __slots__ = ()
+
+    classname = "SiggenRecipeInfo"
+    cachefile = "bb_cache_" + classname +".dat"
+    # we don't want to show this information in graph files so don't set cachefields
+    #cachefields = []
+
+    def __init__(self, filename, metadata):
+        self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False)
+        self.siggen_varvals = metadata.getVar("__siggen_varvals", False)
+        self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False)
+
+    @classmethod
+    def init_cacheData(cls, cachedata):
+        cachedata.siggen_taskdeps = {}
+        cachedata.siggen_gendeps = {}
+        cachedata.siggen_varvals = {}
+
+    def add_cacheData(self, cachedata, fn):
+        cachedata.siggen_gendeps[fn] = self.siggen_gendeps
+        cachedata.siggen_varvals[fn] = self.siggen_varvals
+        cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps
+
+    # The siggen variable data is large and impacts:
+    #  - bitbake's overall memory usage
+    #  - the amount of data sent over IPC between parsing processes and the server
+    #  - the size of the cache files on disk
+    #  - the size of "sigdata" hash information files on disk
+    # The data consists of strings (some large) or frozenset lists of variables
+    # As such, we a) deplicate the data here and b) pass references to the object at second
+    # access (e.g. over IPC or saving into pickle).
+
+    store = {}
+    save_map = {}
+    save_count = 1
+    restore_map = {}
+    restore_count = {}
+
+    @classmethod
+    def reset(cls):
+        # Needs to be called before starting new streamed data in a given process 
+        # (e.g. writing out the cache again)
+        cls.save_map = {}
+        cls.save_count = 1
+        cls.restore_map = {}
+
+    @classmethod
+    def _save(cls, deps):
+        ret = []
+        if not deps:
+            return deps
+        for dep in deps:
+            fs = deps[dep]
+            if fs is None:
+                ret.append((dep, None, None))
+            elif fs in cls.save_map:
+                ret.append((dep, None, cls.save_map[fs]))
+            else:
+                cls.save_map[fs] = cls.save_count
+                ret.append((dep, fs, cls.save_count))
+                cls.save_count = cls.save_count + 1
+        return ret
+
+    @classmethod
+    def _restore(cls, deps, pid):
+        ret = {}
+        if not deps:
+            return deps
+        if pid not in cls.restore_map:
+            cls.restore_map[pid] = {}
+        map = cls.restore_map[pid]
+        for dep, fs, mapnum in deps:
+            if fs is None and mapnum is None:
+                ret[dep] = None
+            elif fs is None:
+                ret[dep] = map[mapnum]
+            else:
+                try:
+                    fs = cls.store[fs]
+                except KeyError:
+                    cls.store[fs] = fs
+                map[mapnum] = fs
+                ret[dep] = fs
+        return ret
+
+    def __getstate__(self):
+        ret = {}
+        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
+            ret[key] = self._save(self.__dict__[key])
+        ret['pid'] = os.getpid()
+        return ret
+
+    def __setstate__(self, state):
+        pid = state['pid']
+        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
+            setattr(self, key, self._restore(state[key], pid))
+
+
 def virtualfn2realfn(virtualfn):
     """
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
     if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
-        elems = virtualfn.split(':')
-        mc = elems[1]
-        virtualfn = ":".join(elems[2:])
+        (_, mc, virtualfn) = virtualfn.split(':', 2)
 
     fn = virtualfn
     cls = ""
@@ -264,7 +367,7 @@ def realfn2virtual(realfn, cls, mc):
 
 def variant2virtual(realfn, variant):
     """
-    Convert a real filename + the associated subclass keyword to a virtual filename
+    Convert a real filename + a variant to a virtual filename
     """
     if variant == "":
         return realfn
@@ -275,96 +378,18 @@ def variant2virtual(realfn, variant):
         return "mc:" + elems[1] + ":" + realfn
     return "virtual:" + variant + ":" + realfn
 
-def parse_recipe(bb_data, bbfile, appends, mc=''):
-    """
-    Parse a recipe
-    """
-
-    chdir_back = False
-
-    bb_data.setVar("__BBMULTICONFIG", mc)
-
-    # expand tmpdir to include this topdir
-    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
-    bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
-    oldpath = os.path.abspath(os.getcwd())
-    bb.parse.cached_mtime_noerror(bbfile_loc)
-
-    # The ConfHandler first looks if there is a TOPDIR and if not
-    # then it would call getcwd().
-    # Previously, we chdir()ed to bbfile_loc, called the handler
-    # and finally chdir()ed back, a couple of thousand times. We now
-    # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
-    if not bb_data.getVar('TOPDIR', False):
-        chdir_back = True
-        bb_data.setVar('TOPDIR', bbfile_loc)
-    try:
-        if appends:
-            bb_data.setVar('__BBAPPEND', " ".join(appends))
-        bb_data = bb.parse.handle(bbfile, bb_data)
-        if chdir_back:
-            os.chdir(oldpath)
-        return bb_data
-    except:
-        if chdir_back:
-            os.chdir(oldpath)
-        raise
-
-
-
-class NoCache(object):
-
-    def __init__(self, databuilder):
-        self.databuilder = databuilder
-        self.data = databuilder.data
-
-    def loadDataFull(self, virtualfn, appends):
-        """
-        Return a complete set of data for fn.
-        To do this, we need to parse the file.
-        """
-        logger.debug("Parsing %s (full)" % virtualfn)
-        (fn, virtual, mc) = virtualfn2realfn(virtualfn)
-        bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
-        return bb_data[virtual]
-
-    def load_bbfile(self, bbfile, appends, virtonly = False, mc=None):
-        """
-        Load and parse one .bb build file
-        Return the data and whether parsing resulted in the file being skipped
-        """
-
-        if virtonly:
-            (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            bb_data.setVar("__ONLYFINALISE", virtual or "default")
-            datastores = parse_recipe(bb_data, bbfile, appends, mc)
-            return datastores
-
-        if mc is not None:
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            return parse_recipe(bb_data, bbfile, appends, mc)
-
-        bb_data = self.data.createCopy()
-        datastores = parse_recipe(bb_data, bbfile, appends)
-
-        for mc in self.databuilder.mcdata:
-            if not mc:
-                continue
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            newstores = parse_recipe(bb_data, bbfile, appends, mc)
-            for ns in newstores:
-                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
-
-        return datastores
-
-class Cache(NoCache):
+#
+# Cooker calls cacheValid on its recipe list, then either calls loadCached
+# from it's main thread or parse from separate processes to generate an up to
+# date cache
+#
+class Cache(object):
     """
     BitBake Cache implementation
     """
     def __init__(self, databuilder, mc, data_hash, caches_array):
-        super().__init__(databuilder)
-        data = databuilder.data
+        self.databuilder = databuilder
+        self.data = databuilder.data
 
         # Pass caches_array information into Cache Constructor
         # It will be used later for deciding whether we
@@ -372,7 +397,7 @@ class Cache(NoCache):
         self.mc = mc
         self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
         self.caches_array = caches_array
-        self.cachedir = data.getVar("CACHE")
+        self.cachedir = self.data.getVar("CACHE")
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
@@ -382,20 +407,12 @@ class Cache(NoCache):
         self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
 
         if self.cachedir in [None, '']:
-            self.has_cache = False
-            self.logger.info("Not using a cache. "
-                             "Set CACHE = <directory> to enable.")
-            return
-
-        self.has_cache = True
+            bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use")
 
     def getCacheFile(self, cachefile):
         return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
 
     def prepare_cache(self, progress):
-        if not self.has_cache:
-            return 0
-
         loaded = 0
 
         self.cachefile = self.getCacheFile("bb_cache.dat")
@@ -434,9 +451,6 @@ class Cache(NoCache):
         return loaded
 
     def cachesize(self):
-        if not self.has_cache:
-            return 0
-
         cachesize = 0
         for cache_class in self.caches_array:
             cachefile = self.getCacheFile(cache_class.cachefile)
@@ -498,11 +512,11 @@ class Cache(NoCache):
 
         return len(self.depends_cache)
 
-    def parse(self, filename, appends):
+    def parse(self, filename, appends, layername):
         """Parse the specified filename, returning the recipe information"""
         self.logger.debug("Parsing %s", filename)
         infos = []
-        datastores = self.load_bbfile(filename, appends, mc=self.mc)
+        datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername)
         depends = []
         variants = []
         # Process the "real" fn last so we can store variants list
@@ -524,43 +538,19 @@ class Cache(NoCache):
 
         return infos
 
-    def load(self, filename, appends):
+    def loadCached(self, filename, appends):
         """Obtain the recipe information for the specified filename,
-        using cached values if available, otherwise parsing.
-
-        Note that if it does parse to obtain the info, it will not
-        automatically add the information to the cache or to your
-        CacheData.  Use the add or add_info method to do so after
-        running this, or use loadData instead."""
-        cached = self.cacheValid(filename, appends)
-        if cached:
-            infos = []
-            # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
-            info_array = self.depends_cache[filename]
-            for variant in info_array[0].variants:
-                virtualfn = variant2virtual(filename, variant)
-                infos.append((virtualfn, self.depends_cache[virtualfn]))
-        else:
-            return self.parse(filename, appends, configdata, self.caches_array)
-
-        return cached, infos
-
-    def loadData(self, fn, appends, cacheData):
-        """Load the recipe info for the specified filename,
-        parsing and adding to the cache if necessary, and adding
-        the recipe information to the supplied CacheData instance."""
-        skipped, virtuals = 0, 0
+        using cached values.
+        """
 
-        cached, infos = self.load(fn, appends)
-        for virtualfn, info_array in infos:
-            if info_array[0].skipped:
-                self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason)
-                skipped += 1
-            else:
-                self.add_info(virtualfn, info_array, cacheData, not cached)
-                virtuals += 1
+        infos = []
+        # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
+        info_array = self.depends_cache[filename]
+        for variant in info_array[0].variants:
+            virtualfn = variant2virtual(filename, variant)
+            infos.append((virtualfn, self.depends_cache[virtualfn]))
 
-        return cached, skipped, virtuals
+        return infos
 
     def cacheValid(self, fn, appends):
         """
@@ -569,10 +559,6 @@ class Cache(NoCache):
         """
         if fn not in self.checked:
             self.cacheValidUpdate(fn, appends)
-
-        # Is cache enabled?
-        if not self.has_cache:
-            return False
         if fn in self.clean:
             return True
         return False
@@ -582,10 +568,6 @@ class Cache(NoCache):
         Is the cache valid for fn?
         Make thorough (slower) checks including timestamps.
         """
-        # Is cache enabled?
-        if not self.has_cache:
-            return False
-
         self.checked.add(fn)
 
         # File isn't in depends_cache
@@ -636,7 +618,7 @@ class Cache(NoCache):
                 for f in flist:
                     if not f:
                         continue
-                    f, exist = f.split(":")
+                    f, exist = f.rsplit(":", 1)
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
                         self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
@@ -692,10 +674,6 @@ class Cache(NoCache):
         Save the cache
         Called from the parser when complete (or exiting)
         """
-
-        if not self.has_cache:
-            return
-
         if self.cacheclean:
             self.logger.debug2("Cache is clean, not saving.")
             return
@@ -716,6 +694,7 @@ class Cache(NoCache):
                             p.dump(info)
 
         del self.depends_cache
+        SiggenRecipeInfo.reset()
 
     @staticmethod
     def mtime(cachefile):
@@ -738,26 +717,11 @@ class Cache(NoCache):
             if watcher:
                 watcher(info_array[0].file_depends)
 
-        if not self.has_cache:
-            return
-
         if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
             if parsed:
                 self.cacheclean = False
             self.depends_cache[filename] = info_array
 
-    def add(self, file_name, data, cacheData, parsed=None):
-        """
-        Save data we need into the cache
-        """
-
-        realfn = virtualfn2realfn(file_name)[0]
-
-        info_array = []
-        for cache_class in self.caches_array:
-            info_array.append(cache_class(realfn, data))
-        self.add_info(file_name, info_array, cacheData, parsed)
-
 class MulticonfigCache(Mapping):
     def __init__(self, databuilder, data_hash, caches_array):
         def progress(p):
@@ -794,6 +758,7 @@ class MulticonfigCache(Mapping):
         loaded = 0
 
         for c in self.__caches.values():
+            SiggenRecipeInfo.reset()
             loaded += c.prepare_cache(progress)
             previous_progress = current_progress
 
@@ -871,11 +836,10 @@ class MultiProcessCache(object):
         self.cachedata = self.create_cachedata()
         self.cachedata_extras = self.create_cachedata()
 
-    def init_cache(self, d, cache_file_name=None):
-        cachedir = (d.getVar("PERSISTENT_DIR") or
-                    d.getVar("CACHE"))
-        if cachedir in [None, '']:
+    def init_cache(self, cachedir, cache_file_name=None):
+        if not cachedir:
             return
+
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
@@ -906,6 +870,10 @@ class MultiProcessCache(object):
         if not self.cachefile:
             return
 
+        have_data = any(self.cachedata_extras)
+        if not have_data:
+            return
+
         glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True)
 
         i = os.getpid()
@@ -940,6 +908,8 @@ class MultiProcessCache(object):
 
         data = self.cachedata
 
+        have_data = False
+
         for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
             f = os.path.join(os.path.dirname(self.cachefile), f)
             try:
@@ -954,12 +924,14 @@ class MultiProcessCache(object):
                 os.unlink(f)
                 continue
 
+            have_data = True
             self.merge_data(extradata, data)
             os.unlink(f)
 
-        with open(self.cachefile, "wb") as f:
-            p = pickle.Pickler(f, -1)
-            p.dump([data, self.__class__.CACHE_VERSION])
+        if have_data:
+            with open(self.cachefile, "wb") as f:
+                p = pickle.Pickler(f, -1)
+                p.dump([data, self.__class__.CACHE_VERSION])
 
         bb.utils.unlockfile(glf)
 
@@ -1015,3 +987,11 @@ class SimpleCache(object):
             p.dump([data, self.cacheversion])
 
         bb.utils.unlockfile(glf)
+
+    def copyfile(self, target):
+        if not self.cachefile:
+            return
+
+        glf = bb.utils.lockfile(self.cachefile + ".lock")
+        shutil.copy(self.cachefile, target)
+        bb.utils.unlockfile(glf)
diff --git a/bitbake/lib/bb/checksum.py b/bitbake/lib/bb/checksum.py
index 1d50a26426..557793d366 100644
--- a/bitbake/lib/bb/checksum.py
+++ b/bitbake/lib/bb/checksum.py
@@ -11,10 +11,13 @@ import os
 import stat
 import bb.utils
 import logging
+import re
 from bb.cache import MultiProcessCache
 
 logger = logging.getLogger("BitBake.Cache")
 
+filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
+
 # mtime cache (non-persistent)
 # based upon the assumption that files do not change during bitbake run
 class FileMtimeCache(object):
@@ -50,6 +53,7 @@ class FileChecksumCache(MultiProcessCache):
         MultiProcessCache.__init__(self)
 
     def get_checksum(self, f):
+        f = os.path.normpath(f)
         entry = self.cachedata[0].get(f)
         cmtime = self.mtime_cache.cached_mtime(f)
         if entry:
@@ -84,22 +88,36 @@ class FileChecksumCache(MultiProcessCache):
                 return None
             return checksum
 
+        #
+        # Changing the format of file-checksums is problematic as both OE and Bitbake have
+        # knowledge of them. We need to encode a new piece of data, the portion of the path
+        # we care about from a checksum perspective. This means that files that change subdirectory
+        # are tracked by the task hashes. To do this, we do something horrible and put a "/./" into
+        # the path. The filesystem handles it but it gives us a marker to know which subsection
+        # of the path to cache.
+        #
         def checksum_dir(pth):
             # Handle directories recursively
             if pth == "/":
                 bb.fatal("Refusing to checksum /")
+            pth = pth.rstrip("/")
             dirchecksums = []
             for root, dirs, files in os.walk(pth, topdown=True):
                 [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
                 for name in files:
-                    fullpth = os.path.join(root, name)
+                    fullpth = os.path.join(root, name).replace(pth, os.path.join(pth, "."))
                     checksum = checksum_file(fullpth)
                     if checksum:
                         dirchecksums.append((fullpth, checksum))
             return dirchecksums
 
         checksums = []
-        for pth in filelist.split():
+        for pth in filelist_regex.split(filelist):
+            if not pth:
+                continue
+            pth = pth.strip()
+            if not pth:
+                continue
             exist = pth.split(":")[1]
             if exist == "False":
                 continue
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 25a7ac69d3..2e8b7ced3c 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -25,6 +27,7 @@ import ast
 import sys
 import codegen
 import logging
+import inspect
 import bb.pysh as pysh
 import bb.utils, bb.data
 import hashlib
@@ -56,10 +59,40 @@ def check_indent(codestr):
 
     return codestr
 
-# A custom getstate/setstate using tuples is actually worth 15% cachesize by
-# avoiding duplication of the attribute names!
+modulecode_deps = {}
 
+def add_module_functions(fn, functions, namespace):
+    import os
+    fstat = os.stat(fn)
+    fixedhash = fn + ":" + str(fstat.st_size) +  ":" + str(fstat.st_mtime)
+    for f in functions:
+        name = "%s.%s" % (namespace, f)
+        parser = PythonParser(name, logger)
+        try:
+            parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f)
+            #bb.warn("Cached %s" % f)
+        except KeyError:
+            lines, lineno = inspect.getsourcelines(functions[f])
+            src = "".join(lines)
+            parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f)
+            #bb.warn("Not cached %s" % f)
+        execs = parser.execs.copy()
+        # Expand internal module exec references
+        for e in parser.execs:
+            if e in functions:
+                execs.remove(e)
+                execs.add(namespace + "." + e)
+        modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()]
+        #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains))
+
+def update_module_dependencies(d):
+    for mod in modulecode_deps:
+        excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split())
+        if excludes:
+            modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3]]
 
+# A custom getstate/setstate using tuples is actually worth 15% cachesize by
+# avoiding duplication of the attribute names!
 class SetCache(object):
     def __init__(self):
         self.setcache = {}
@@ -152,12 +185,12 @@ class CodeParserCache(MultiProcessCache):
         self.shellcachelines[h] = cacheline
         return cacheline
 
-    def init_cache(self, d):
+    def init_cache(self, cachedir):
         # Check if we already have the caches
         if self.pythoncache:
             return
 
-        MultiProcessCache.init_cache(self, d)
+        MultiProcessCache.init_cache(self, cachedir)
 
         # cachedata gets re-assigned in the parent
         self.pythoncache = self.cachedata[0]
@@ -169,8 +202,8 @@ class CodeParserCache(MultiProcessCache):
 
 codeparsercache = CodeParserCache()
 
-def parser_cache_init(d):
-    codeparsercache.init_cache(d)
+def parser_cache_init(cachedir):
+    codeparsercache.init_cache(cachedir)
 
 def parser_cache_save():
     codeparsercache.save_extras()
@@ -195,6 +228,10 @@ class BufferedLogger(Logger):
                 self.target.handle(record)
         self.buffer = []
 
+class DummyLogger():
+    def flush(self):
+        return
+
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
@@ -212,26 +249,26 @@ class PythonParser():
             funcstr = codegen.to_source(func)
             argstr = codegen.to_source(arg)
         except TypeError:
-            self.log.debug(2, 'Failed to convert function and argument to source form')
+            self.log.debug2('Failed to convert function and argument to source form')
         else:
-            self.log.debug(1, self.unhandled_message % (funcstr, argstr))
+            self.log.debug(self.unhandled_message % (funcstr, argstr))
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
         if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
-            if isinstance(node.args[0], ast.Str):
-                varname = node.args[0].s
-                if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
+            if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
+                varname = node.args[0].value
+                if name in self.containsfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].add(node.args[1].s)
-                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str):
+                    self.contains[varname].add(node.args[1].value)
+                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].update(node.args[1].s.split())
+                    self.contains[varname].update(node.args[1].value.split())
                 elif name.endswith(self.getvarflags):
-                    if isinstance(node.args[1], ast.Str):
-                        self.references.add('%s[%s]' % (varname, node.args[1].s))
+                    if isinstance(node.args[1], ast.Constant):
+                        self.references.add('%s[%s]' % (varname, node.args[1].value))
                     else:
                         self.warn(node.func, node.args[1])
                 else:
@@ -239,8 +276,8 @@ class PythonParser():
             else:
                 self.warn(node.func, node.args[0])
         elif name and name.endswith(".expand"):
-            if isinstance(node.args[0], ast.Str):
-                value = node.args[0].s
+            if isinstance(node.args[0], ast.Constant):
+                value = node.args[0].value
                 d = bb.data.init()
                 parser = d.expandWithRefs(value, self.name)
                 self.references |= parser.references
@@ -250,8 +287,8 @@ class PythonParser():
                         self.contains[varname] = set()
                     self.contains[varname] |= parser.contains[varname]
         elif name in self.execfuncs:
-            if isinstance(node.args[0], ast.Str):
-                self.var_execs.add(node.args[0].s)
+            if isinstance(node.args[0], ast.Constant):
+                self.var_execs.add(node.args[0].value)
             else:
                 self.warn(node.func, node.args[0])
         elif name and isinstance(node.func, (ast.Name, ast.Attribute)):
@@ -276,16 +313,24 @@ class PythonParser():
         self.contains = {}
         self.execs = set()
         self.references = set()
-        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
 
         self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
 
-    def parse_python(self, node, lineno=0, filename="<string>"):
-        if not node or not node.strip():
+    # For the python module code it is expensive to have the function text so it is
+    # uses a different fixedhash to cache against. We can take the hit on obtaining the
+    # text if it isn't in the cache.
+    def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None):
+        if not fixedhash and (not node or not node.strip()):
             return
 
-        h = bbhash(str(node))
+        if fixedhash:
+            h = fixedhash
+        else:
+            h = bbhash(str(node))
 
         if h in codeparsercache.pythoncache:
             self.references = set(codeparsercache.pythoncache[h].refs)
@@ -303,6 +348,12 @@ class PythonParser():
                 self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
             return
 
+        if fixedhash and not node:
+            raise KeyError
+
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log)
+
         # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
         node = "\n" * int(lineno) + node
         code = compile(check_indent(str(node)), filename, "exec",
@@ -321,7 +372,11 @@ class ShellParser():
         self.funcdefs = set()
         self.allexecs = set()
         self.execs = set()
-        self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
+        self._name = name
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
+
         self.unhandled_template = "unable to handle non-literal command '%s'"
         self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template)
 
@@ -340,6 +395,9 @@ class ShellParser():
             self.execs = set(codeparsercache.shellcacheextras[h].execs)
             return self.execs
 
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log)
+
         self._parse_shell(value)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 
@@ -450,7 +508,7 @@ class ShellParser():
 
                 cmd = word[1]
                 if cmd.startswith("$"):
-                    self.log.debug(1, self.unhandled_template % cmd)
+                    self.log.debug(self.unhandled_template % cmd)
                 elif cmd == "eval":
                     command = " ".join(word for _, word in words[1:])
                     self._parse_shell(command)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index dd77cdd6e2..1fcb9bf14c 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -20,6 +20,7 @@ Commands are queued in a CommandQueue
 
 from collections import OrderedDict, defaultdict
 
+import io
 import bb.event
 import bb.cooker
 import bb.remotedata
@@ -50,23 +51,32 @@ class Command:
     """
     A queue of asynchronous commands for bitbake
     """
-    def __init__(self, cooker):
+    def __init__(self, cooker, process_server):
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
         self.remotedatastores = None
 
-        # FIXME Add lock for this
+        self.process_server = process_server
+        # Access with locking using process_server.{get/set/clear}_async_cmd()
         self.currentAsyncCommand = None
 
-    def runCommand(self, commandline, ro_only = False):
+    def runCommand(self, commandline, process_server, ro_only=False):
         command = commandline.pop(0)
 
         # Ensure cooker is ready for commands
-        if command != "updateConfig" and command != "setFeatures":
-            self.cooker.init_configdata()
-            if not self.remotedatastores:
-                self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
+        if command not in ["updateConfig", "setFeatures", "ping"]:
+            try:
+                self.cooker.init_configdata()
+                if not self.remotedatastores:
+                    self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
+            except (Exception, SystemExit) as exc:
+                import traceback
+                if isinstance(exc, bb.BBHandledException):
+                    # We need to start returning real exceptions here. Until we do, we can't
+                    # tell if an exception is an instance of bb.BBHandledException
+                    return None, "bb.BBHandledException()\n" + traceback.format_exc()
+                return None, traceback.format_exc()
 
         if hasattr(CommandsSync, command):
             # Can run synchronous commands straight away
@@ -75,7 +85,6 @@ class Command:
                 if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
-                self.cooker.process_inotify_updates()
                 if getattr(command_method, 'needconfig', True):
                     self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
@@ -90,24 +99,23 @@ class Command:
                 return None, traceback.format_exc()
             else:
                 return result, None
-        if self.currentAsyncCommand is not None:
-            return None, "Busy (%s in progress)" % self.currentAsyncCommand[0]
         if command not in CommandsAsync.__dict__:
             return None, "No such command"
-        self.currentAsyncCommand = (command, commandline)
-        self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker)
+        if not process_server.set_async_cmd((command, commandline)):
+            return None, "Busy (%s in progress)" % self.process_server.get_async_cmd()[0]
+        self.cooker.idleCallBackRegister(self.runAsyncCommand, process_server)
         return True, None
 
-    def runAsyncCommand(self):
+    def runAsyncCommand(self, _, process_server, halt):
         try:
-            self.cooker.process_inotify_updates()
             if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit
                 self.cooker.updateCache()
-                return False
-            if self.currentAsyncCommand is not None:
-                (command, options) = self.currentAsyncCommand
+                return bb.server.process.idleFinish("Cooker in error state")
+            cmd = process_server.get_async_cmd()
+            if cmd is not None:
+                (command, options) = cmd
                 commandmethod = getattr(CommandsAsync, command)
                 needcache = getattr( commandmethod, "needcache" )
                 if needcache and self.cooker.state != bb.cooker.state.running:
@@ -117,24 +125,21 @@ class Command:
                     commandmethod(self.cmds_async, self, options)
                     return False
             else:
-                return False
+                return bb.server.process.idleFinish("Nothing to do, no async command?")
         except KeyboardInterrupt as exc:
-            self.finishAsyncCommand("Interrupted")
-            return False
+            return bb.server.process.idleFinish("Interrupted")
         except SystemExit as exc:
             arg = exc.args[0]
             if isinstance(arg, str):
-                self.finishAsyncCommand(arg)
+                return bb.server.process.idleFinish(arg)
             else:
-                self.finishAsyncCommand("Exited with %s" % arg)
-            return False
+                return bb.server.process.idleFinish("Exited with %s" % arg)
         except Exception as exc:
             import traceback
             if isinstance(exc, bb.BBHandledException):
-                self.finishAsyncCommand("")
+                return bb.server.process.idleFinish("")
             else:
-                self.finishAsyncCommand(traceback.format_exc())
-            return False
+                return bb.server.process.idleFinish(traceback.format_exc())
 
     def finishAsyncCommand(self, msg=None, code=None):
         if msg or msg == "":
@@ -143,8 +148,8 @@ class Command:
             bb.event.fire(CommandExit(code), self.cooker.data)
         else:
             bb.event.fire(CommandCompleted(), self.cooker.data)
-        self.currentAsyncCommand = None
         self.cooker.finishcommand()
+        self.process_server.clear_async_cmd()
 
     def reset(self):
         if self.remotedatastores:
@@ -157,6 +162,14 @@ class CommandsSync:
     These must not influence any running synchronous command.
     """
 
+    def ping(self, command, params):
+        """
+        Allow a UI to check the server is still alive
+        """
+        return "Still alive!"
+    ping.needconfig = False
+    ping.readonly = True
+
     def stateShutdown(self, command, params):
         """
         Trigger cooker 'shutdown' mode
@@ -294,6 +307,11 @@ class CommandsSync:
         return ret
     getLayerPriorities.readonly = True
 
+    def revalidateCaches(self, command, params):
+        """Called by UI clients when metadata may have changed"""
+        command.cooker.revalidateCaches()
+    parseConfiguration.needconfig = False
+
     def getRecipes(self, command, params):
         try:
             mc = params[0]
@@ -500,6 +518,17 @@ class CommandsSync:
         d = command.remotedatastores[dsindex].varhistory
         return getattr(d, method)(*args, **kwargs)
 
+    def dataStoreConnectorVarHistCmdEmit(self, command, params):
+        dsindex = params[0]
+        var = params[1]
+        oval = params[2]
+        val = params[3]
+        d = command.remotedatastores[params[4]]
+
+        o = io.StringIO()
+        command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d)
+        return o.getvalue()
+
     def dataStoreConnectorIncHistCmd(self, command, params):
         dsindex = params[0]
         method = params[1]
@@ -521,8 +550,8 @@ class CommandsSync:
         and return a datastore object representing the environment
         for the recipe.
         """
-        fn = params[0]
-        mc = bb.runqueue.mc_from_tid(fn)
+        virtualfn = params[0]
+        (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
@@ -537,6 +566,7 @@ class CommandsSync:
                 appendfiles = command.cooker.collections[mc].get_file_appends(fn)
         else:
             appendfiles = []
+        layername = command.cooker.collections[mc].calc_bbfile_priority(fn)[2]
         # We are calling bb.cache locally here rather than on the server,
         # but that's OK because it doesn't actually need anything from
         # the server barring the global datastore (which we have a remote
@@ -544,11 +574,10 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
+            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)[cls]
         else:
             # Use the standard path
-            parser = bb.cache.NoCache(command.cooker.databuilder)
-            envdata = parser.loadDataFull(fn, appendfiles)
+            envdata = command.cooker.databuilder.parseRecipe(virtualfn, appendfiles, layername)
         idx = command.remotedatastores.store(envdata)
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
@@ -647,6 +676,16 @@ class CommandsAsync:
         command.finishAsyncCommand()
     findFilesMatchingInDir.needcache = False
 
+    def testCookerCommandEvent(self, command, params):
+        """
+        Dummy command used by OEQA selftest to test tinfoil without IO
+        """
+        pattern = params[0]
+
+        command.cooker.testCookerCommandEvent(pattern)
+        command.finishAsyncCommand()
+    testCookerCommandEvent.needcache = False
+
     def findConfigFilePath(self, command, params):
         """
         Find the path of the requested configuration file
@@ -711,7 +750,7 @@ class CommandsAsync:
         """
         event = params[0]
         bb.event.fire(eval(event), command.cooker.data)
-        command.currentAsyncCommand = None
+        process_server.clear_async_cmd()
     triggerEvent.needcache = False
 
     def resetCooker(self, command, params):
@@ -738,7 +777,14 @@ class CommandsAsync:
         (mc, pn) = bb.runqueue.split_mc(params[0])
         taskname = params[1]
         sigs = params[2]
+        bb.siggen.check_siggen_version(bb.siggen)
         res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc])
         bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()
     findSigInfo.needcache = False
+
+    def getTaskSignatures(self, command, params):
+        res = command.cooker.getTaskSignatures(params[0], params[1])
+        bb.event.fire(bb.event.GetTaskSignatureResult(res), command.cooker.data)
+        command.finishAsyncCommand()
+    getTaskSignatures.needcache = True
diff --git a/bitbake/lib/bb/compress/_pipecompress.py b/bitbake/lib/bb/compress/_pipecompress.py
new file mode 100644
index 0000000000..4a403d62cf
--- /dev/null
+++ b/bitbake/lib/bb/compress/_pipecompress.py
@@ -0,0 +1,196 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+    cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+    """
+    Open a compressed file in binary or text mode.
+
+    Users should not call this directly. A specific compression library can use
+    this helper to provide it's own "open" command
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+    "rb".
+
+    For binary mode, this function is equivalent to the cls constructor:
+    cls(filename, mode). In this case, the encoding, errors and newline
+    arguments must not be provided.
+
+    For text mode, a cls object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    file_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes, os.PathLike, int)):
+        binary_file = cls(filename, file_mode, **kwargs)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(
+            binary_file, encoding, errors, newline, write_through=True
+        )
+    else:
+        return binary_file
+
+
+class CompressionError(OSError):
+    pass
+
+
+class PipeFile(io.RawIOBase):
+    """
+    Class that implements generically piping to/from a compression program
+
+    Derived classes should add the function get_compress() and get_decompress()
+    that return the required commands. Input will be piped into stdin and the
+    (de)compressed output should be written to stdout, e.g.:
+
+        class FooFile(PipeCompressionFile):
+            def get_decompress(self):
+                return ["fooc", "--decompress", "--stdout"]
+
+            def get_compress(self):
+                return ["fooc", "--compress", "--stdout"]
+
+    """
+
+    READ = 0
+    WRITE = 1
+
+    def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+        if "t" in mode or "U" in mode:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if not "b" in mode:
+            mode += "b"
+
+        if mode.startswith("r"):
+            self.mode = self.READ
+        elif mode.startswith("w"):
+            self.mode = self.WRITE
+        else:
+            raise ValueError("Invalid mode %r" % mode)
+
+        if fileobj is not None:
+            self.fileobj = fileobj
+        else:
+            self.fileobj = builtins.open(filename, mode or "rb")
+
+        if self.mode == self.READ:
+            self.p = subprocess.Popen(
+                self.get_decompress(),
+                stdin=self.fileobj,
+                stdout=subprocess.PIPE,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdout
+        else:
+            self.p = subprocess.Popen(
+                self.get_compress(),
+                stdin=subprocess.PIPE,
+                stdout=self.fileobj,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdin
+
+        self.__closed = False
+
+    def _check_process(self):
+        if self.p is None:
+            return
+
+        returncode = self.p.wait()
+        if returncode:
+            raise CompressionError("Process died with %d" % returncode)
+        self.p = None
+
+    def close(self):
+        if self.closed:
+            return
+
+        self.pipe.close()
+        if self.p is not None:
+            self._check_process()
+        self.fileobj.close()
+
+        self.__closed = True
+
+    @property
+    def closed(self):
+        return self.__closed
+
+    def fileno(self):
+        return self.pipe.fileno()
+
+    def flush(self):
+        self.pipe.flush()
+
+    def isatty(self):
+        return self.pipe.isatty()
+
+    def readable(self):
+        return self.mode == self.READ
+
+    def writable(self):
+        return self.mode == self.WRITE
+
+    def readinto(self, b):
+        if self.mode != self.READ:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+            )
+        size = self.pipe.readinto(b)
+        if size == 0:
+            self._check_process()
+        return size
+
+    def write(self, data):
+        if self.mode != self.WRITE:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+            )
+        data = self.pipe.write(data)
+
+        if not data:
+            self._check_process()
+
+        return data
diff --git a/bitbake/lib/bb/compress/lz4.py b/bitbake/lib/bb/compress/lz4.py
new file mode 100644
index 0000000000..88b0989322
--- /dev/null
+++ b/bitbake/lib/bb/compress/lz4.py
@@ -0,0 +1,19 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb.compress._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]
diff --git a/bitbake/lib/bb/compress/zstd.py b/bitbake/lib/bb/compress/zstd.py
new file mode 100644
index 0000000000..cdbbe9d60f
--- /dev/null
+++ b/bitbake/lib/bb/compress/zstd.py
@@ -0,0 +1,30 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb.compress._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs):
+        self.num_threads = num_threads
+        self.compresslevel = compresslevel
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c", "-%d" % self.compresslevel]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index f4ab797edf..c5bfef55d6 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -13,7 +13,6 @@ import sys, os, glob, os.path, re, time
 import itertools
 import logging
 import multiprocessing
-import sre_constants
 import threading
 from io import StringIO, UnsupportedOperation
 from contextlib import closing
@@ -23,7 +22,6 @@ from bb import utils, data, parse, event, cache, providers, taskdata, runqueue,
 import queue
 import signal
 import prserv.serv
-import pyinotify
 import json
 import pickle
 import codecs
@@ -81,7 +79,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3))
+    _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS, RECIPE_SIGGEN_INFO] = list(range(4))
 
     def __init__(self):
         self._features=set()
@@ -104,12 +102,15 @@ class CookerFeatures(object):
 
 class EventWriter:
     def __init__(self, cooker, eventfile):
-        self.file_inited = None
         self.cooker = cooker
         self.eventfile = eventfile
         self.event_queue = []
 
-    def write_event(self, event):
+    def write_variables(self):
+        with open(self.eventfile, "a") as f:
+            f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
+
+    def send(self, event):
         with open(self.eventfile, "a") as f:
             try:
                 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
@@ -119,28 +120,6 @@ class EventWriter:
                 import traceback
                 print(err, traceback.format_exc())
 
-    def send(self, event):
-        if self.file_inited:
-            # we have the file, just write the event
-            self.write_event(event)
-        else:
-            # init on bb.event.BuildStarted
-            name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
-                with open(self.eventfile, "w") as f:
-                    f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
-                self.file_inited = True
-
-                # write pending events
-                for evt in self.event_queue:
-                    self.write_event(evt)
-
-                # also write the current event
-                self.write_event(event)
-            else:
-                # queue all events until the file is inited
-                self.event_queue.append(event)
 
 #============================================================================#
 # BBCooker
@@ -150,8 +129,10 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, featureSet=None, idleCallBackRegister=None):
+    def __init__(self, featureSet=None, server=None):
         self.recipecaches = None
+        self.baseconfig_valid = False
+        self.parsecache_valid = False
         self.eventlog = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
@@ -159,34 +140,22 @@ class BBCooker:
             for f in featureSet:
                 self.featureset.setFeature(f)
 
+        self.orig_syspath = sys.path.copy()
+        self.orig_sysmodules = [*sys.modules]
+
         self.configuration = bb.cookerdata.CookerConfiguration()
 
-        self.idleCallBackRegister = idleCallBackRegister
+        self.process_server = server
+        self.idleCallBackRegister = None
+        self.waitIdle = None
+        if server:
+            self.idleCallBackRegister = server.register_idle_function
+            self.waitIdle = server.wait_for_idle
 
         bb.debug(1, "BBCooker starting %s" % time.time())
-        sys.stdout.flush()
-
-        self.configwatcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
-        sys.stdout.flush()
-
-        self.configwatcher.bbseen = set()
-        self.configwatcher.bbwatchedfiles = set()
-        self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
-        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
-        sys.stdout.flush()
-        self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
-                         pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
-                         pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
-        self.watcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
-        sys.stdout.flush()
-        self.watcher.bbseen = set()
-        self.watcher.bbwatchedfiles = set()
-        self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
-
-        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
-        sys.stdout.flush()
+
+        self.configwatched = {}
+        self.parsewatched = {}
 
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
@@ -197,14 +166,6 @@ class BBCooker:
         self.hashserv = None
         self.hashservaddr = None
 
-        self.inotify_modified_files = []
-
-        def _process_inotify_updates(server, cooker, abort):
-            cooker.process_inotify_updates()
-            return 1.0
-
-        self.idleCallBackRegister(_process_inotify_updates, self)
-
         # TOSTOP must not be set or our children will hang when they output
         try:
             fd = sys.stdout.fileno()
@@ -218,7 +179,7 @@ class BBCooker:
         except UnsupportedOperation:
             pass
 
-        self.command = bb.command.Command(self)
+        self.command = bb.command.Command(self, self.process_server)
         self.state = state.initial
 
         self.parser = None
@@ -228,84 +189,37 @@ class BBCooker:
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
         bb.debug(1, "BBCooker startup complete %s" % time.time())
-        sys.stdout.flush()
 
     def init_configdata(self):
         if not hasattr(self, "data"):
             self.initConfigurationData()
             bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
-            sys.stdout.flush()
             self.handlePRServ()
 
-    def process_inotify_updates(self):
-        for n in [self.confignotifier, self.notifier]:
-            if n.check_events(timeout=0):
-                # read notified events and enqeue them
-                n.read_events()
-                n.process_events()
-
-    def config_notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self.parsecache_valid = False
-            self.baseconfig_valid = False
-            bb.parse.clear_cache()
-            return
-        if not event.pathname in self.configwatcher.bbwatchedfiles:
-            return
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self.baseconfig_valid = False
-
-    def notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self.parsecache_valid = False
-            bb.parse.clear_cache()
-            return
-        if event.pathname.endswith("bitbake-cookerdaemon.log") \
-                or event.pathname.endswith("bitbake.lock"):
-            return
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self.parsecache_valid = False
+    def _baseconfig_set(self, value):
+        if value and not self.baseconfig_valid:
+            bb.server.process.serverlog("Base config valid")
+        elif not value and self.baseconfig_valid:
+            bb.server.process.serverlog("Base config invalidated")
+        self.baseconfig_valid = value
+
+    def _parsecache_set(self, value):
+        if value and not self.parsecache_valid:
+            bb.server.process.serverlog("Parse cache valid")
+        elif not value and self.parsecache_valid:
+            bb.server.process.serverlog("Parse cache invalidated")
+        self.parsecache_valid = value
+
+    def add_filewatch(self, deps, configwatcher=False):
+        if configwatcher:
+            watcher = self.configwatched
+        else:
+            watcher = self.parsewatched
 
-    def add_filewatch(self, deps, watcher=None, dirs=False):
-        if not watcher:
-            watcher = self.watcher
         for i in deps:
-            watcher.bbwatchedfiles.add(i[0])
-            if dirs:
-                f = i[0]
-            else:
-                f = os.path.dirname(i[0])
-            if f in watcher.bbseen:
-                continue
-            watcher.bbseen.add(f)
-            watchtarget = None
-            while True:
-                # We try and add watches for files that don't exist but if they did, would influence
-                # the parser. The parent directory of these files may not exist, in which case we need
-                # to watch any parent that does exist for changes.
-                try:
-                    watcher.add_watch(f, self.watchmask, quiet=False)
-                    if watchtarget:
-                        watcher.bbwatchedfiles.add(watchtarget)
-                    break
-                except pyinotify.WatchManagerError as e:
-                    if 'ENOENT' in str(e):
-                        watchtarget = f
-                        f = os.path.dirname(f)
-                        if f in watcher.bbseen:
-                            break
-                        watcher.bbseen.add(f)
-                        continue
-                    if 'ENOSPC' in str(e):
-                        providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
-                        providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
-                        providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
-                        providerlog.error("Root privilege is required to modify max_user_watches.")
-                    raise
+            f = i[0]
+            mtime = i[1]
+            watcher[f] = mtime
 
     def sigterm_exception(self, signum, stackframe):
         if signum == signal.SIGTERM:
@@ -313,6 +227,7 @@ class BBCooker:
         elif signum == signal.SIGHUP:
             bb.warn("Cooker received SIGHUP, shutting down...")
         self.state = state.forceshutdown
+        bb.event._should_exit.set()
 
     def setFeatures(self, features):
         # we only accept a new feature set if we're in state initial, so we can reset without problems
@@ -330,6 +245,13 @@ class BBCooker:
         self.state = state.initial
         self.caches_array = []
 
+        sys.path = self.orig_syspath.copy()
+        for mod in [*sys.modules]:
+            if mod not in self.orig_sysmodules:
+                del sys.modules[mod]
+
+        self.configwatched = {}
+
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
         if hasattr(self, "data"):
@@ -338,12 +260,12 @@ class BBCooker:
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.enableDataTracking()
 
-        all_extra_cache_names = []
+        caches_name_array = ['bb.cache:CoreRecipeInfo']
         # We hardcode all known cache types in a single place, here.
         if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
-            all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
-
-        caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
+            caches_name_array.append("bb.cache_extra:HobRecipeInfo")
+        if CookerFeatures.RECIPE_SIGGEN_INFO in self.featureset:
+            caches_name_array.append("bb.cache:SiggenRecipeInfo")
 
         # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
         # This is the entry point, no further check needed!
@@ -362,6 +284,10 @@ class BBCooker:
         self.data_hash = self.databuilder.data_hash
         self.extraconfigdata = {}
 
+        eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG")
+        if not self.configuration.writeeventlog and eventlog:
+            self.setupEventLog(eventlog)
+
         if consolelog:
             self.data.setVar("BB_CONSOLELOG", consolelog)
 
@@ -371,31 +297,42 @@ class BBCooker:
             self.disableDataTracking()
 
         for mc in self.databuilder.mcdata.values():
-            mc.renameVar("__depends", "__base_depends")
-            self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
+            self.add_filewatch(mc.getVar("__base_depends", False), configwatcher=True)
 
-        self.baseconfig_valid = True
-        self.parsecache_valid = False
+        self._baseconfig_set(True)
+        self._parsecache_set(False)
 
     def handlePRServ(self):
         # Setup a PR Server based on the new configuration
         try:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError as e:
-            bb.fatal("Unable to start PR Server, exitting")
+            bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log")
 
         if self.data.getVar("BB_HASHSERVE") == "auto":
             # Create a new hash server bound to a unix domain socket
             if not self.hashserv:
                 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+                upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
+                if upstream:
+                    import socket
+                    try:
+                        sock = socket.create_connection(upstream.split(":"), 5)
+                        sock.close()
+                    except socket.error as e:
+                        bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
+                                 % (upstream, repr(e)))
+
                 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
-                self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
-                self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
-                self.hashserv.process.start()
-            self.data.setVar("BB_HASHSERVE", self.hashservaddr)
-            self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
-            self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr)
+                self.hashserv = hashserv.create_server(
+                    self.hashservaddr,
+                    dbfile,
+                    sync=False,
+                    upstream=upstream,
+                )
+                self.hashserv.serve_as_process(log_level=logging.WARNING)
             for mc in self.databuilder.mcdata:
+                self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
                 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
 
         bb.parse.init_parser(self.data)
@@ -410,6 +347,29 @@ class BBCooker:
         if hasattr(self, "data"):
             self.data.disableTracking()
 
+    def revalidateCaches(self):
+        bb.parse.clear_cache()
+
+        clean = True
+        for f in self.configwatched:
+            if not bb.parse.check_mtime(f, self.configwatched[f]):
+                bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                self._baseconfig_set(False)
+                self._parsecache_set(False)
+                clean = False
+                break
+
+        if clean:
+            for f in self.parsewatched:
+                if not bb.parse.check_mtime(f, self.parsewatched[f]):
+                    bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                    self._parsecache_set(False)
+                    clean = False
+                    break
+
+        if not clean:
+            bb.parse.BBHandler.cached_statements = {}
+
     def parseConfiguration(self):
         self.updateCacheSync()
 
@@ -428,8 +388,24 @@ class BBCooker:
             self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
         self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
-
-        self.parsecache_valid = False
+        self.collections = {}
+        for mc in self.multiconfigs:
+            self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+
+        self._parsecache_set(False)
+
+    def setupEventLog(self, eventlog):
+        if self.eventlog and self.eventlog[0] != eventlog:
+            bb.event.unregister_UIHhandler(self.eventlog[1])
+            self.eventlog = None
+        if not self.eventlog or self.eventlog[0] != eventlog:
+            # we log all events to a file if so directed
+            # register the log file writer as UI Handler
+            if not os.path.exists(os.path.dirname(eventlog)):
+                bb.utils.mkdirhier(os.path.dirname(eventlog))
+            writer = EventWriter(self, eventlog)
+            EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
+            self.eventlog = (eventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)), writer)
 
     def updateConfigOpts(self, options, environment, cmdline):
         self.ui_cmdline = cmdline
@@ -450,14 +426,7 @@ class BBCooker:
                 setattr(self.configuration, o, options[o])
 
         if self.configuration.writeeventlog:
-            if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
-                bb.event.unregister_UIHhandler(self.eventlog[1])
-            if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
-                # we log all events to a file if so directed
-                # register the log file writer as UI Handler
-                writer = EventWriter(self, self.configuration.writeeventlog)
-                EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
-                self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
+            self.setupEventLog(self.configuration.writeeventlog)
 
         bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
         bb.msg.loggerDefaultDomains = self.configuration.debug_domains
@@ -487,37 +456,37 @@ class BBCooker:
         # Now update all the variables not in the datastore to match
         self.configuration.env = environment
 
+        self.revalidateCaches()
         if not clean:
             logger.debug("Base environment change, triggering reparse")
             self.reset()
 
-    def runCommands(self, server, data, abort):
-        """
-        Run any queued asynchronous command
-        This is done by the idle handler so it runs in true context rather than
-        tied to any UI.
-        """
-
-        return self.command.runAsyncCommand()
-
     def showVersions(self):
 
-        (latest_versions, preferred_versions) = self.findProviders()
+        (latest_versions, preferred_versions, required) = self.findProviders()
 
-        logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
-        logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
+        logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
+        logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
 
         for p in sorted(self.recipecaches[''].pkg_pn):
-            pref = preferred_versions[p]
+            preferred = preferred_versions[p]
             latest = latest_versions[p]
+            requiredstr = ""
+            preferredstr = ""
+            if required[p]:
+                if preferred[0] is not None:
+                    requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
+                else:
+                    bb.fatal("REQUIRED_VERSION of package %s not available" % p)
+            else:
+                preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
 
-            prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
 
-            if pref == latest:
-                prefstr = ""
+            if preferred == latest:
+                preferredstr = ""
 
-            logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
+            logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
 
     def showEnvironment(self, buildfile=None, pkgs_to_build=None):
         """
@@ -533,6 +502,8 @@ class BBCooker:
         if not orig_tracking:
             self.enableDataTracking()
             self.reset()
+            # reset() resets to the UI requested value so we have to redo this
+            self.enableDataTracking()
 
         def mc_base(p):
             if p.startswith('mc:'):
@@ -556,21 +527,21 @@ class BBCooker:
                 if pkgs_to_build[0] in set(ignore.split()):
                     bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
+                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True)
 
                 mc = runlist[0][0]
                 fn = runlist[0][3]
 
         if fn:
             try:
-                bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
-                envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
+                layername = self.collections[mc].calc_bbfile_priority(fn)[2]
+                envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn), layername)
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
         else:
             if not mc in self.databuilder.mcdata:
-                bb.fatal('Not multiconfig named "%s" found' % mc)
+                bb.fatal('No multiconfig named "%s" found' % mc)
             envdata = self.databuilder.mcdata[mc]
             data.expandKeys(envdata)
             parse.ast.runAnonFuncs(envdata)
@@ -585,7 +556,7 @@ class BBCooker:
             data.emit_env(env, envdata, True)
             logger.plain(env.getvalue())
 
-        # emit the metadata which isnt valid shell
+        # emit the metadata which isn't valid shell
         for e in sorted(envdata.keys()):
             if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
                 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
@@ -594,7 +565,7 @@ class BBCooker:
             self.disableDataTracking()
             self.reset()
 
-    def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False):
+    def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False):
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
@@ -641,7 +612,7 @@ class BBCooker:
         localdata = {}
 
         for mc in self.multiconfigs:
-            taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
+            taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete)
             localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
             bb.data.expandKeys(localdata[mc])
 
@@ -690,19 +661,18 @@ class BBCooker:
                     taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
                     mcdeps |= set(taskdata[mc].get_mcdepends())
                 new = False
-                for mc in self.multiconfigs:
-                    for k in mcdeps:
-                        if k in seen:
-                            continue
-                        l = k.split(':')
-                        depmc = l[2]
-                        if depmc not in self.multiconfigs:
-                            bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
-                        else:
-                            logger.debug("Adding providers for multiconfig dependency %s" % l[3])
-                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
-                            seen.add(k)
-                            new = True
+                for k in mcdeps:
+                    if k in seen:
+                        continue
+                    l = k.split(':')
+                    depmc = l[2]
+                    if depmc not in self.multiconfigs:
+                        bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
+                    else:
+                        logger.debug("Adding providers for multiconfig dependency %s" % l[3])
+                        taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                        seen.add(k)
+                        new = True
 
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
@@ -715,7 +685,7 @@ class BBCooker:
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
 
-        # We set abort to False here to prevent unbuildable targets raising
+        # We set halt to False here to prevent unbuildable targets raising
         # an exception when we're just generating data
         taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
 
@@ -792,7 +762,9 @@ class BBCooker:
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
-                depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
+                if depmc:
+                    depmc = "mc:" + depmc + ":"
+                depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
                 packages = []
@@ -1056,6 +1028,11 @@ class BBCooker:
         if matches:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
+    def testCookerCommandEvent(self, filepattern):
+        # Dummy command used by OEQA selftest to test tinfoil without IO
+        matches = ["A", "B"]
+        bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
+
     def findProviders(self, mc=''):
         return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
 
@@ -1063,10 +1040,16 @@ class BBCooker:
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
             eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
-            filename = eligible[0]
+            if eligible is not None:
+                filename = eligible[0]
+            else:
+                filename = None
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            if required and preferred_file is None:
+                return None, None, None, None
+            return (latest, latest_f, preferred_ver, preferred_file)
         else:
             return None, None, None, None
 
@@ -1211,15 +1194,15 @@ class BBCooker:
                                     except bb.utils.VersionStringException as vse:
                                         bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
                                     if not res:
-                                        parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
+                                        parselog.debug3("Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
                                         continue
                                 else:
-                                    parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
+                                    parselog.debug3("Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
                                     continue
-                            parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec)
+                            parselog.debug3("Layer '%s' recommends layer '%s', so we are adding it", c, rec)
                             collection_depends[c].append(rec)
                         else:
-                            parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
+                            parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
 
             # Recursively work out collection priorities based on dependencies
             def calc_layer_priority(collection):
@@ -1231,7 +1214,7 @@ class BBCooker:
                         if depprio > max_depprio:
                             max_depprio = depprio
                     max_depprio += 1
-                    parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio)
+                    parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio)
                     collection_priorities[collection] = max_depprio
 
             # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
@@ -1243,7 +1226,7 @@ class BBCooker:
                     errors = True
                     continue
                 elif regex == "":
-                    parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
+                    parselog.debug("BBFILE_PATTERN_%s is empty" % c)
                     cre = re.compile('^NULL$')
                     errors = False
                 else:
@@ -1290,8 +1273,8 @@ class BBCooker:
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
-        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+        collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1357,7 +1340,8 @@ class BBCooker:
 
         bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
 
-        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
+        layername = self.collections[mc].calc_bbfile_priority(fn)[2]
+        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername)
         infos = dict(infos)
 
         fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1383,14 +1367,16 @@ class BBCooker:
         self.recipecaches[mc].rundeps[fn] = defaultdict(list)
         self.recipecaches[mc].runrecs[fn] = defaultdict(list)
 
+        bb.parse.siggen.setup_datacache(self.recipecaches)
+
         # Invalidate task for target if force mode active
         if self.configuration.force:
             logger.verbose("Invalidate task %s, %s", task, fn)
-            bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
+            bb.parse.siggen.invalidate_task(task, fn)
 
         # Setup taskdata structure
         taskdata = {}
-        taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
+        taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt)
         taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
 
         if quietlog:
@@ -1400,17 +1386,20 @@ class BBCooker:
         buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
         if fireevents:
             bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
+            if self.eventlog:
+                self.eventlog[2].write_variables()
+            bb.event.enable_heartbeat()
 
         # Execute the runqueue
         runlist = [[mc, item, task, fn]]
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
 
-        def buildFileIdle(server, rq, abort):
+        def buildFileIdle(server, rq, halt):
 
             msg = None
             interrupted = 0
-            if abort or self.state == state.forceshutdown:
+            if halt or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
@@ -1425,37 +1414,68 @@ class BBCooker:
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
-                self.command.finishAsyncCommand(str(exc))
                 if quietlog:
                     bb.runqueue.logger.setLevel(rqloglevel)
-                return False
+                return bb.server.process.idleFinish(str(exc))
 
             if not retval:
                 if fireevents:
                     bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
-                self.command.finishAsyncCommand(msg)
+                    bb.event.disable_heartbeat()
                 # We trashed self.recipecaches above
-                self.parsecache_valid = False
+                self._parsecache_set(False)
                 self.configuration.limited_deps = False
                 bb.parse.siggen.reset(self.data)
                 if quietlog:
                     bb.runqueue.logger.setLevel(rqloglevel)
-                return False
+                return bb.server.process.idleFinish(msg)
             if retval is True:
                 return True
             return retval
 
         self.idleCallBackRegister(buildFileIdle, rq)
 
+    def getTaskSignatures(self, target, tasks):
+        sig = []
+        getAllTaskSignatures = False
+
+        if not tasks:
+            tasks = ["do_build"]
+            getAllTaskSignatures = True
+
+        for task in tasks:
+            taskdata, runlist = self.buildTaskData(target, task, self.configuration.halt)
+            rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
+            rq.rqdata.prepare()
+
+            for l in runlist:
+                mc, pn, taskname, fn = l
+
+                taskdep = rq.rqdata.dataCaches[mc].task_deps[fn]
+                for t in taskdep['tasks']:
+                    if t in taskdep['nostamp'] or "setscene" in t:
+                        continue
+                    tid = bb.runqueue.build_tid(mc, fn, t)
+
+                    if t in task or getAllTaskSignatures:
+                        try:
+                            rq.rqdata.prepare_task_hash(tid)
+                            sig.append([pn, t, rq.rqdata.get_task_unihash(tid)])
+                        except KeyError:
+                            sig.append(self.getTaskSignatures(target, [t])[0])
+
+        return sig
+
     def buildTargets(self, targets, task):
         """
         Attempt to build the targets specified
         """
 
-        def buildTargetsIdle(server, rq, abort):
+        def buildTargetsIdle(server, rq, halt):
             msg = None
             interrupted = 0
-            if abort or self.state == state.forceshutdown:
+            if halt or self.state == state.forceshutdown:
+                bb.event._should_exit.set()
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
@@ -1470,16 +1490,16 @@ class BBCooker:
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
-                self.command.finishAsyncCommand(str(exc))
-                return False
+                return bb.server.process.idleFinish(str(exc))
 
             if not retval:
                 try:
                     for mc in self.multiconfigs:
                         bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
                 finally:
-                    self.command.finishAsyncCommand(msg)
-                return False
+                    bb.event.disable_heartbeat()
+                return bb.server.process.idleFinish(msg)
+
             if retval is True:
                 return True
             return retval
@@ -1498,7 +1518,7 @@ class BBCooker:
 
         bb.event.fire(bb.event.BuildInit(packages), self.data)
 
-        taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
+        taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt)
 
         buildname = self.data.getVar("BUILDNAME", False)
 
@@ -1511,6 +1531,9 @@ class BBCooker:
 
         for mc in self.multiconfigs:
             bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
+        if self.eventlog:
+            self.eventlog[2].write_variables()
+        bb.event.enable_heartbeat()
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         if 'universe' in targets:
@@ -1520,7 +1543,13 @@ class BBCooker:
 
 
     def getAllKeysWithFlags(self, flaglist):
+        def dummy_autorev(d):
+            return
+
         dump = {}
+        # Horrible but for now we need to avoid any sideeffects of autorev being called
+        saved = bb.fetch2.get_autorev
+        bb.fetch2.get_autorev = dummy_autorev
         for k in self.data.keys():
             try:
                 expand = True
@@ -1540,6 +1569,7 @@ class BBCooker:
                             dump[k][d] = None
             except Exception as e:
                 print(e)
+        bb.fetch2.get_autorev = saved
         return dump
 
 
@@ -1547,13 +1577,6 @@ class BBCooker:
         if self.state == state.running:
             return
 
-        # reload files for which we got notifications
-        for p in self.inotify_modified_files:
-            bb.parse.update_cache(p)
-            if p in bb.parse.BBHandler.cached_statements:
-                del bb.parse.BBHandler.cached_statements[p]
-        self.inotify_modified_files = []
-
         if not self.baseconfig_valid:
             logger.debug("Reloading base configuration data")
             self.initConfigurationData()
@@ -1566,7 +1589,7 @@ class BBCooker:
 
         if self.state in (state.shutdown, state.forceshutdown, state.error):
             if hasattr(self.parser, 'shutdown'):
-                self.parser.shutdown(clean=False, force = True)
+                self.parser.shutdown(clean=False)
                 self.parser.final_cleanup()
             raise bb.BBHandledException()
 
@@ -1574,6 +1597,9 @@ class BBCooker:
             self.updateCacheSync()
 
         if self.state != state.parsing and not self.parsecache_valid:
+            bb.server.process.serverlog("Parsing started")
+            self.parsewatched = {}
+
             bb.parse.siggen.reset(self.data)
             self.parseConfiguration ()
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
@@ -1587,30 +1613,27 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collections = {}
-
             mcfilelist = {}
             total_masked = 0
             searchdirs = set()
             for mc in self.multiconfigs:
-                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
                 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
 
                 mcfilelist[mc] = filelist
                 total_masked += masked
                 searchdirs |= set(search)
 
-            # Add inotify watches for directories searched for bb/bbappend files
+            # Add mtimes for directories searched for bb/bbappend files
             for dirent in searchdirs:
-                self.add_filewatch([[dirent]], dirs=True)
+                self.add_filewatch([(dirent, bb.parse.cached_mtime_noerror(dirent))])
 
             self.parser = CookerParser(self, mcfilelist, total_masked)
-            self.parsecache_valid = True
+            self._parsecache_set(True)
 
         self.state = state.parsing
 
         if not self.parser.parse_next():
-            collectlog.debug(1, "parsing complete")
+            collectlog.debug("parsing complete")
             if self.parser.error:
                 raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
@@ -1633,7 +1656,7 @@ class BBCooker:
         # Return a copy, don't modify the original
         pkgs_to_build = pkgs_to_build[:]
 
-        if len(pkgs_to_build) == 0:
+        if not pkgs_to_build:
             raise NothingToBuild
 
         ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
@@ -1655,7 +1678,7 @@ class BBCooker:
 
         if 'universe' in pkgs_to_build:
             parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
-            parselog.debug(1, "collating packages for \"universe\"")
+            parselog.debug("collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
                 for t in self.recipecaches[mc].universe_target:
@@ -1680,26 +1703,36 @@ class BBCooker:
     def post_serve(self):
         self.shutdown(force=True)
         prserv.serv.auto_shutdown()
+        if hasattr(bb.parse, "siggen"):
+            bb.parse.siggen.exit()
         if self.hashserv:
             self.hashserv.process.terminate()
             self.hashserv.process.join()
         if hasattr(self, "data"):
             bb.event.fire(CookerExit(), self.data)
 
-    def shutdown(self, force = False):
+    def shutdown(self, force=False):
         if force:
             self.state = state.forceshutdown
+            bb.event._should_exit.set()
         else:
             self.state = state.shutdown
 
         if self.parser:
-            self.parser.shutdown(clean=not force, force=force)
+            self.parser.shutdown(clean=False)
             self.parser.final_cleanup()
 
     def finishcommand(self):
+        if hasattr(self.parser, 'shutdown'):
+            self.parser.shutdown(clean=False)
+            self.parser.final_cleanup()
         self.state = state.initial
+        bb.event._should_exit.clear()
 
     def reset(self):
+        if hasattr(bb.parse, "siggen"):
+            bb.parse.siggen.exit()
+        self.finishcommand()
         self.initConfigurationData()
         self.handlePRServ()
 
@@ -1711,9 +1744,9 @@ class BBCooker:
         if hasattr(self, "data"):
            self.databuilder.reset()
            self.data = self.databuilder.data
-        self.parsecache_valid = False
-        self.baseconfig_valid = False
-
+        # In theory tinfoil could have modified the base data before parsing,
+        # ideally need to track if anything did modify the datastore
+        self._parsecache_set(False)
 
 class CookerExit(bb.event.Event):
     """
@@ -1728,16 +1761,16 @@ class CookerCollectFiles(object):
     def __init__(self, priorities, mc=''):
         self.mc = mc
         self.bbappends = []
-        # Priorities is a list of tupples, with the second element as the pattern.
+        # Priorities is a list of tuples, with the second element as the pattern.
         # We need to sort the list with the longest pattern first, and so on to
         # the shortest.  This allows nested layers to be properly evaluated.
         self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
     def calc_bbfile_priority(self, filename):
-        for _, _, regex, pri in self.bbfile_config_priorities:
+        for layername, _, regex, pri in self.bbfile_config_priorities:
             if regex.match(filename):
-                return pri, regex
-        return 0, None
+                return pri, regex, layername
+        return 0, None, None
 
     def get_bbfiles(self):
         """Get list of default .bb files by reading out the current directory"""
@@ -1756,7 +1789,7 @@ class CookerCollectFiles(object):
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
+            found += [os.path.join(dir, f) for f in files if (f.endswith(('.bb', '.bbappend')))]
 
         return found
 
@@ -1764,7 +1797,7 @@ class CookerCollectFiles(object):
         """Collect all available .bb build files"""
         masked = 0
 
-        collectlog.debug(1, "collecting .bb files")
+        collectlog.debug("collecting .bb files")
 
         files = (config.getVar( "BBFILES") or "").split()
 
@@ -1772,14 +1805,14 @@ class CookerCollectFiles(object):
         files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
         config.setVar("BBFILES_PRIORITIZED", " ".join(files))
 
-        if not len(files):
+        if not files:
             files = self.get_bbfiles()
 
-        if not len(files):
+        if not files:
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
             bb.event.fire(CookerExit(), eventdata)
 
-        # We need to track where we look so that we can add inotify watches. There
+        # We need to track where we look so that we can know when the cache is invalid. There
         # is no nice way to do this, this is horrid. We intercept the os.listdir()
         # (or os.scandir() for python 3.6+) calls while we run glob().
         origlistdir = os.listdir
@@ -1835,7 +1868,7 @@ class CookerCollectFiles(object):
                 try:
                     re.compile(mask)
                     bbmasks.append(mask)
-                except sre_constants.error:
+                except re.error:
                     collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask)
 
             # Then validate the combined regular expressions. This should never
@@ -1843,7 +1876,7 @@ class CookerCollectFiles(object):
             bbmask = "|".join(bbmasks)
             try:
                 bbmask_compiled = re.compile(bbmask)
-            except sre_constants.error:
+            except re.error:
                 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask)
                 bbmask = None
 
@@ -1851,7 +1884,7 @@ class CookerCollectFiles(object):
         bbappend = []
         for f in newfiles:
             if bbmask and bbmask_compiled.search(f):
-                collectlog.debug(1, "skipping masked file %s", f)
+                collectlog.debug("skipping masked file %s", f)
                 masked += 1
                 continue
             if f.endswith('.bb'):
@@ -1859,7 +1892,7 @@ class CookerCollectFiles(object):
             elif f.endswith('.bbappend'):
                 bbappend.append(f)
             else:
-                collectlog.debug(1, "skipping %s: unknown file extension", f)
+                collectlog.debug("skipping %s: unknown file extension", f)
 
         # Build a list of .bbappend files for each .bb file
         for f in bbappend:
@@ -1910,7 +1943,7 @@ class CookerCollectFiles(object):
         # Calculate priorities for each file
         for p in pkgfns:
             realfn, cls, mc = bb.cache.virtualfn2realfn(p)
-            priorities[p], regex = self.calc_bbfile_priority(realfn)
+            priorities[p], regex, _ = self.calc_bbfile_priority(realfn)
             if regex in unmatched_regex:
                 matched_regex.add(regex)
                 unmatched_regex.remove(regex)
@@ -1961,15 +1994,30 @@ class ParsingFailure(Exception):
         Exception.__init__(self, realexception, recipe)
 
 class Parser(multiprocessing.Process):
-    def __init__(self, jobs, results, quit, init, profile):
+    def __init__(self, jobs, results, quit, profile):
         self.jobs = jobs
         self.results = results
         self.quit = quit
-        self.init = init
         multiprocessing.Process.__init__(self)
         self.context = bb.utils.get_context().copy()
         self.handlers = bb.event.get_class_handlers().copy()
         self.profile = profile
+        self.queue_signals = False
+        self.signal_received = []
+        self.signal_threadlock = threading.Lock()
+
+    def catch_sig(self, signum, frame):
+        if self.queue_signals:
+            self.signal_received.append(signum)
+        else:
+            self.handle_sig(signum, frame)
+
+    def handle_sig(self, signum, frame):
+        if signum == signal.SIGTERM:
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            os.kill(os.getpid(), signal.SIGTERM)
+        elif signum == signal.SIGINT:
+            signal.default_int_handler(signum, frame)
 
     def run(self):
 
@@ -1989,38 +2037,50 @@ class Parser(multiprocessing.Process):
             prof.dump_stats(logfile)
 
     def realrun(self):
-        if self.init:
-            self.init()
+        # Signal handling here is hard. We must not terminate any process or thread holding the write
+        # lock for the event stream as it will not be released, ever, and things will hang.
+        # Python handles signals in the main thread/process but they can be raised from any thread and
+        # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section
+        # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any
+        # new thread should also do so) and we defer handling but we handle with the local thread lock
+        # held (a threading lock, not a multiprocessing one) so that no other thread in the process
+        # can be in the critical section.
+        signal.signal(signal.SIGTERM, self.catch_sig)
+        signal.signal(signal.SIGHUP, signal.SIG_DFL)
+        signal.signal(signal.SIGINT, self.catch_sig)
+        bb.utils.set_process_name(multiprocessing.current_process().name)
+        multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
+        multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
 
         pending = []
-        while True:
-            try:
-                self.quit.get_nowait()
-            except queue.Empty:
-                pass
-            else:
-                self.results.close()
-                self.results.join_thread()
-                break
+        havejobs = True
+        try:
+            while havejobs or pending:
+                if self.quit.is_set():
+                    break
 
-            if pending:
-                result = pending.pop()
-            else:
+                job = None
                 try:
                     job = self.jobs.pop()
                 except IndexError:
-                    self.results.close()
-                    self.results.join_thread()
-                    break
-                result = self.parse(*job)
-                # Clear the siggen cache after parsing to control memory usage, its huge
-                bb.parse.siggen.postparsing_clean_cache()
-            try:
-                self.results.put(result, timeout=0.25)
-            except queue.Full:
-                pending.append(result)
+                    havejobs = False
+                if job:
+                    result = self.parse(*job)
+                    # Clear the siggen cache after parsing to control memory usage, its huge
+                    bb.parse.siggen.postparsing_clean_cache()
+                    pending.append(result)
+
+                if pending:
+                    try:
+                        result = pending.pop()
+                        self.results.put(result, timeout=0.05)
+                    except queue.Full:
+                        pending.append(result)
+        finally:
+            self.results.close()
+            self.results.join_thread()
 
-    def parse(self, mc, cache, filename, appends):
+    def parse(self, mc, cache, filename, appends, layername):
         try:
             origfilter = bb.event.LogHandler.filter
             # Record the filename we're parsing into any events generated
@@ -2034,17 +2094,17 @@ class Parser(multiprocessing.Process):
             bb.event.set_class_handlers(self.handlers.copy())
             bb.event.LogHandler.filter = parse_filter
 
-            return True, mc, cache.parse(filename, appends)
+            return True, mc, cache.parse(filename, appends, layername)
         except Exception as exc:
             tb = sys.exc_info()[2]
             exc.recipe = filename
             exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
-            return True, exc
+            return True, None, exc
         # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
         # and for example a worker thread doesn't just exit on its own in response to
         # a SystemExit event for example.
         except BaseException as exc:
-            return True, ParsingFailure(exc, filename)
+            return True, None, ParsingFailure(exc, filename)
         finally:
             bb.event.LogHandler.filter = origfilter
 
@@ -2074,10 +2134,11 @@ class CookerParser(object):
         for mc in self.cooker.multiconfigs:
             for filename in self.mcfilelist[mc]:
                 appends = self.cooker.collections[mc].get_file_appends(filename)
+                layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2]
                 if not self.bb_caches[mc].cacheValid(filename, appends):
-                    self.willparse.add((mc, self.bb_caches[mc], filename, appends))
+                    self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername))
                 else:
-                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
+                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername))
 
         self.total = len(self.fromcache) + len(self.willparse)
         self.toparse = len(self.willparse)
@@ -2086,6 +2147,7 @@ class CookerParser(object):
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
                                  multiprocessing.cpu_count()), self.toparse)
 
+        bb.cache.SiggenRecipeInfo.reset()
         self.start()
         self.haveshutdown = False
         self.syncthread = None
@@ -2095,15 +2157,8 @@ class CookerParser(object):
         self.processes = []
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
-            def init():
-                signal.signal(signal.SIGTERM, signal.SIG_DFL)
-                signal.signal(signal.SIGHUP, signal.SIG_DFL)
-                signal.signal(signal.SIGINT, signal.SIG_IGN)
-                bb.utils.set_process_name(multiprocessing.current_process().name)
-                multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
-                multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
-
-            self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
+
+            self.parser_quit = multiprocessing.Event()
             self.result_queue = multiprocessing.Queue()
 
             def chunkify(lst,n):
@@ -2111,14 +2166,14 @@ class CookerParser(object):
             self.jobs = chunkify(list(self.willparse), self.num_processes)
 
             for i in range(0, self.num_processes):
-                parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
+                parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile)
                 parser.start()
                 self.process_names.append(parser.name)
                 self.processes.append(parser)
 
             self.results = itertools.chain(self.results, self.parse_generator())
 
-    def shutdown(self, clean=True, force=False):
+    def shutdown(self, clean=True, eventmsg="Parsing halted due to errors"):
         if not self.toparse:
             return
         if self.haveshutdown:
@@ -2132,9 +2187,9 @@ class CookerParser(object):
                                             self.total)
 
             bb.event.fire(event, self.cfgdata)
-
-        for process in self.processes:
-            self.parser_quit.put(None)
+        else:
+            bb.event.fire(bb.event.ParseError(eventmsg), self.cfgdata)
+            bb.error("Parsing halted due to errors, see error messages above")
 
         # Cleanup the queue before call process.join(), otherwise there might be
         # deadlocks.
@@ -2144,25 +2199,39 @@ class CookerParser(object):
             except queue.Empty:
                 break
 
-        for process in self.processes:
-            if force:
-                process.join(.1)
-                process.terminate()
-            else:
-                process.join()
-
-        self.parser_quit.close()
-        # Allow data left in the cancel queue to be discarded
-        self.parser_quit.cancel_join_thread()
-
         def sync_caches():
             for c in self.bb_caches.values():
+                bb.cache.SiggenRecipeInfo.reset()
                 c.sync()
 
-        sync = threading.Thread(target=sync_caches, name="SyncThread")
-        self.syncthread = sync
-        sync.start()
+        self.syncthread = threading.Thread(target=sync_caches, name="SyncThread")
+        self.syncthread.start()
+
+        self.parser_quit.set()
+
+        for process in self.processes:
+            process.join(0.5)
+
+        for process in self.processes:
+            if process.exitcode is None:
+                os.kill(process.pid, signal.SIGINT)
+
+        for process in self.processes:
+            process.join(0.5)
+
+        for process in self.processes:
+            if process.exitcode is None:
+                process.terminate()
+
+        for process in self.processes:
+            process.join()
+            # Added in 3.7, cleans up zombies
+            if hasattr(process, "close"):
+                process.close()
+
+        bb.codeparser.parser_cache_save()
         bb.codeparser.parser_cache_savemerge()
+        bb.cache.SiggenRecipeInfo.reset()
         bb.fetch.fetcher_parse_done()
         if self.cooker.configuration.profile:
             profiles = []
@@ -2180,49 +2249,64 @@ class CookerParser(object):
             self.syncthread.join()
 
     def load_cached(self):
-        for mc, cache, filename, appends in self.fromcache:
-            cached, infos = cache.load(filename, appends)
-            yield not cached, mc, infos
+        for mc, cache, filename, appends, layername in self.fromcache:
+            infos = cache.loadCached(filename, appends)
+            yield False, mc, infos
 
     def parse_generator(self):
-        while True:
+        empty = False
+        while self.processes or not empty:
+            for process in self.processes.copy():
+                if not process.is_alive():
+                    process.join()
+                    self.processes.remove(process)
+
             if self.parsed >= self.toparse:
                 break
 
             try:
                 result = self.result_queue.get(timeout=0.25)
             except queue.Empty:
-                pass
+                empty = True
+                yield None, None, None
             else:
-                value = result[1]
-                if isinstance(value, BaseException):
-                    raise value
-                else:
-                    yield result
+                empty = False
+                yield result
+
+        if not (self.parsed >= self.toparse):
+            raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
+
 
     def parse_next(self):
         result = []
         parsed = None
         try:
             parsed, mc, result = next(self.results)
+            if isinstance(result, BaseException):
+                # Turn exceptions back into exceptions
+                raise result
+            if parsed is None:
+                # Timeout, loop back through the main loop
+                return True
+
         except StopIteration:
             self.shutdown()
             return False
         except bb.BBHandledException as exc:
             self.error += 1
-            logger.error('Failed to parse recipe: %s' % exc.recipe)
-            self.shutdown(clean=False, force=True)
+            logger.debug('Failed to parse recipe: %s' % exc.recipe)
+            self.shutdown(clean=False)
             return False
         except ParsingFailure as exc:
             self.error += 1
             logger.error('Unable to parse %s: %s' %
                      (exc.recipe, bb.exceptions.to_string(exc.realexception)))
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False)
             return False
         except bb.parse.ParseError as exc:
             self.error += 1
             logger.error(str(exc))
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False, eventmsg=str(exc))
             return False
         except bb.data_smart.ExpansionError as exc:
             self.error += 1
@@ -2231,7 +2315,7 @@ class CookerParser(object):
             tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
             logger.error('ExpansionError during parsing %s', value.recipe,
                          exc_info=(etype, value, tb))
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False)
             return False
         except Exception as exc:
             self.error += 1
@@ -2243,7 +2327,7 @@ class CookerParser(object):
                 # Most likely, an exception occurred during raising an exception
                 import traceback
                 logger.error('Exception during parse: %s' % traceback.format_exc())
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False)
             return False
 
         self.current += 1
@@ -2265,11 +2349,13 @@ class CookerParser(object):
         return True
 
     def reparse(self, filename):
+        bb.cache.SiggenRecipeInfo.reset()
         to_reparse = set()
         for mc in self.cooker.multiconfigs:
-            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+            layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2]
+            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename), layername))
 
-        for mc, filename, appends in to_reparse:
-            infos = self.bb_caches[mc].parse(filename, appends)
+        for mc, filename, appends, layername in to_reparse:
+            infos = self.bb_caches[mc].parse(filename, appends, layername)
             for vfn, info_array in infos:
                 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 1c1e008c6b..0649e40995 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -57,7 +57,7 @@ class ConfigParameters(object):
 
     def updateToServer(self, server, environment):
         options = {}
-        for o in ["abort", "force", "invalidate_stamp",
+        for o in ["halt", "force", "invalidate_stamp",
                   "dry_run", "dump_signatures",
                   "extra_assume_provided", "profile",
                   "prefile", "postfile", "server_timeout",
@@ -86,7 +86,7 @@ class ConfigParameters(object):
                 action['msg'] = "Only one target can be used with the --environment option."
             elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
                 action['msg'] = "No target should be used with the --environment and --buildfile options."
-            elif len(self.options.pkgs_to_build) > 0:
+            elif self.options.pkgs_to_build:
                 action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
             else:
                 action['action'] = ["showEnvironment", self.options.buildfile]
@@ -124,7 +124,7 @@ class CookerConfiguration(object):
         self.prefile = []
         self.postfile = []
         self.cmd = None
-        self.abort = True
+        self.halt = True
         self.force = False
         self.profile = False
         self.nosetscene = False
@@ -160,12 +160,7 @@ def catch_parse_error(func):
     def wrapped(fn, *args):
         try:
             return func(fn, *args)
-        except IOError as exc:
-            import traceback
-            parselog.critical(traceback.format_exc())
-            parselog.critical("Unable to parse %s: %s" % (fn, exc))
-            raise bb.BBHandledException()
-        except bb.data_smart.ExpansionError as exc:
+        except Exception as exc:
             import traceback
 
             bbdir = os.path.dirname(__file__) + os.sep
@@ -177,14 +172,11 @@ def catch_parse_error(func):
                     break
             parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
             raise bb.BBHandledException()
-        except bb.parse.ParseError as exc:
-            parselog.critical(str(exc))
-            raise bb.BBHandledException()
     return wrapped
 
 @catch_parse_error
 def parse_config_file(fn, data, include=True):
-    return bb.parse.handle(fn, data, include)
+    return bb.parse.handle(fn, data, include, baseconfig=True)
 
 @catch_parse_error
 def _inherit(bbclass, data):
@@ -210,7 +202,7 @@ def findConfigFile(configfile, data):
 
 #
 # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
-# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
+# up to /. If that fails, bitbake would fall back to cwd.
 #
 
 def findTopdir():
@@ -223,11 +215,8 @@ def findTopdir():
     layerconf = findConfigFile("bblayers.conf", d)
     if layerconf:
         return os.path.dirname(os.path.dirname(layerconf))
-    if bbpath:
-        bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
-        if bitbakeconf:
-            return os.path.dirname(os.path.dirname(bitbakeconf))
-    return None
+
+    return os.path.abspath(os.getcwd())
 
 class CookerDataBuilder(object):
 
@@ -250,10 +239,14 @@ class CookerDataBuilder(object):
         self.savedenv = bb.data.init()
         for k in cookercfg.env:
             self.savedenv.setVar(k, cookercfg.env[k])
+            if k in bb.data_smart.bitbake_renamed_vars:
+                bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
+                bb.fatal("Exiting to allow enviroment variables to be corrected")
 
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
         self.basedata.setVar("BB_ORIGENV", self.savedenv)
+        self.basedata.setVar("__bbclasstype", "global")
 
         if worker:
             self.basedata.setVar("BB_WORKERCONTEXT", "1")
@@ -261,15 +254,15 @@ class CookerDataBuilder(object):
         self.data = self.basedata
         self.mcdata = {}
 
-    def parseBaseConfiguration(self):
+    def parseBaseConfiguration(self, worker=False):
+        mcdata = {}
         data_hash = hashlib.sha256()
         try:
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
-            if self.data.getVar("BB_WORKERCONTEXT", False) is None:
+            if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
                 bb.fetch.fetcher_init(self.data)
             bb.parse.init_parser(self.data)
-            bb.codeparser.parser_cache_init(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
 
@@ -287,38 +280,62 @@ class CookerDataBuilder(object):
 
             bb.parse.init_parser(self.data)
             data_hash.update(self.data.get_hash().encode('utf-8'))
-            self.mcdata[''] = self.data
+            mcdata[''] = self.data
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
-                mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
-                bb.event.fire(bb.event.ConfigParsed(), mcdata)
-                self.mcdata[config] = mcdata
-                data_hash.update(mcdata.get_hash().encode('utf-8'))
+                if config[0].isdigit():
+                    bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
+                parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
+                bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
+                mcdata[config] = parsed_mcdata
+                data_hash.update(parsed_mcdata.get_hash().encode('utf-8'))
             if multiconfig:
-                bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
+                bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
 
             self.data_hash = data_hash.hexdigest()
-        except (SyntaxError, bb.BBHandledException):
-            raise bb.BBHandledException()
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
             raise bb.BBHandledException()
-        except Exception:
-            logger.exception("Error parsing configuration files")
+
+        bb.codeparser.update_module_dependencies(self.data)
+
+        # Handle obsolete variable names
+        d = self.data
+        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
+        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
+        issues = False
+        for v in renamedvars:
+            if d.getVar(v) != None or d.hasOverrides(v):
+                issues = True
+                loginfo = {}
+                history = d.varhistory.get_variable_refs(v)
+                for h in history:
+                    for line in history[h]:
+                        loginfo = {'file' : h, 'line' : line}
+                        bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
+                if not history:
+                    bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
+        if issues:
             raise bb.BBHandledException()
 
+        for mc in mcdata:
+            mcdata[mc].renameVar("__depends", "__base_depends")
+            mcdata[mc].setVar("__bbclasstype", "recipe")
+
         # Create a copy so we can reset at a later date when UIs disconnect
-        self.origdata = self.data
-        self.data = bb.data.createCopy(self.origdata)
-        self.mcdata[''] = self.data
+        self.mcorigdata = mcdata
+        for mc in mcdata:
+            self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
+        self.data = self.mcdata['']
 
     def reset(self):
         # We may not have run parseBaseConfiguration() yet
-        if not hasattr(self, 'origdata'):
+        if not hasattr(self, 'mcorigdata'):
             return
-        self.data = bb.data.createCopy(self.origdata)
-        self.mcdata[''] = self.data
+        for mc in self.mcorigdata:
+            self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc])
+        self.data = self.mcdata['']
 
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
@@ -333,15 +350,23 @@ class CookerDataBuilder(object):
 
         layerconf = self._findLayerConf(data)
         if layerconf:
-            parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
+            parselog.debug2("Found bblayers.conf (%s)", layerconf)
             # By definition bblayers.conf is in conf/ of TOPDIR.
             # We may have been called with cwd somewhere else so reset TOPDIR
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
+            if not data.getVar("BB_CACHEDIR"):
+                data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+
+            bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
+
             layers = (data.getVar('BBLAYERS') or "").split()
             broken_layers = []
 
+            if not layers:
+                bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition")
+
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
 
@@ -357,8 +382,10 @@ class CookerDataBuilder(object):
                 parselog.critical("Please check BBLAYERS in %s" % (layerconf))
                 raise bb.BBHandledException()
 
+            layerseries = None
+            compat_entries = {}
             for layer in layers:
-                parselog.debug(2, "Adding layer %s", layer)
+                parselog.debug2("Adding layer %s", layer)
                 if 'HOME' in approved and '~' in layer:
                     layer = os.path.expanduser(layer)
                 if layer.endswith('/'):
@@ -369,8 +396,27 @@ class CookerDataBuilder(object):
                 data.expandVarref('LAYERDIR')
                 data.expandVarref('LAYERDIR_RE')
 
+                # Sadly we can't have nice things.
+                # Some layers think they're going to be 'clever' and copy the values from
+                # another layer, e.g. using ${LAYERSERIES_COMPAT_core}. The whole point of
+                # this mechanism is to make it clear which releases a layer supports and
+                # show when a layer master branch is bitrotting and is unmaintained.
+                # We therefore avoid people doing this here.
+                collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
+                for c in collections:
+                    compat_entry = data.getVar("LAYERSERIES_COMPAT_%s" % c)
+                    if compat_entry:
+                        compat_entries[c] = set(compat_entry.split())
+                        data.delVar("LAYERSERIES_COMPAT_%s" % c)
+                if not layerseries:
+                    layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
+                    if layerseries:
+                        data.delVar("LAYERSERIES_CORENAMES")
+
             data.delVar('LAYERDIR_RE')
             data.delVar('LAYERDIR')
+            for c in compat_entries:
+                data.setVar("LAYERSERIES_COMPAT_%s" % c, " ".join(sorted(compat_entries[c])))
 
             bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
             collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
@@ -389,26 +435,38 @@ class CookerDataBuilder(object):
             if invalid:
                 bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
-            layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
             collections_tmp = collections[:]
             for c in collections:
                 collections_tmp.remove(c)
                 if c in collections_tmp:
                     bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
-                compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
+
+                compat = set()
+                if c in compat_entries:
+                    compat = compat_entries[c]
+                if compat and not layerseries:
+                    bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
                 if compat and not (compat & layerseries):
                     bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
                               % (c, " ".join(layerseries), " ".join(compat)))
                 elif not compat and not data.getVar("BB_WORKERCONTEXT"):
                     bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
 
+            data.setVar("LAYERSERIES_CORENAMES", " ".join(sorted(layerseries)))
+
         if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"
             if not layerconf:
                 msg += (" and bitbake did not find a conf/bblayers.conf file in"
                         " the expected location.\nMaybe you accidentally"
                         " invoked bitbake from the wrong directory?")
-            raise SystemExit(msg)
+            bb.fatal(msg)
+
+        if not data.getVar("TOPDIR"):
+            data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
+        if not data.getVar("BB_CACHEDIR"):
+            data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+        bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
 
         data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
 
@@ -421,7 +479,7 @@ class CookerDataBuilder(object):
         for bbclass in bbclasses:
             data = _inherit(bbclass, data)
 
-        # Nomally we only register event handlers at the end of parsing .bb files
+        # Normally we only register event handlers at the end of parsing .bb files
         # We register any handlers we've found so far here...
         for var in data.getVar('__BBHANDLERS', False) or []:
             handlerfn = data.getVarFlag(var, "filename", False)
@@ -435,3 +493,54 @@ class CookerDataBuilder(object):
 
         return data
 
+    @staticmethod
+    def _parse_recipe(bb_data, bbfile, appends, mc, layername):
+        bb_data.setVar("__BBMULTICONFIG", mc)
+        bb_data.setVar("FILE_LAYERNAME", layername)
+
+        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
+        bb.parse.cached_mtime_noerror(bbfile_loc)
+
+        if appends:
+            bb_data.setVar('__BBAPPEND', " ".join(appends))
+
+        return bb.parse.handle(bbfile, bb_data)
+
+    def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None):
+        """
+        Load and parse one .bb build file
+        Return the data and whether parsing resulted in the file being skipped
+        """
+
+        if virtonly:
+            (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile)
+            bb_data = self.mcdata[mc].createCopy()
+            bb_data.setVar("__ONLYFINALISE", virtual or "default")
+            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+
+        if mc is not None:
+            bb_data = self.mcdata[mc].createCopy()
+            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+
+        bb_data = self.data.createCopy()
+        datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername)
+
+        for mc in self.mcdata:
+            if not mc:
+                continue
+            bb_data = self.mcdata[mc].createCopy()
+            newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+            for ns in newstores:
+                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
+
+        return datastores
+
+    def parseRecipe(self, virtualfn, appends, layername):
+        """
+        Return a complete set of data for fn.
+        To do this, we need to parse the file.
+        """
+        logger.debug("Parsing %s (full)" % virtualfn)
+        (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn)
+        datastores = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
+        return datastores[virtual]
diff --git a/bitbake/lib/bb/daemonize.py b/bitbake/lib/bb/daemonize.py
index c187fcfc6c..7689404436 100644
--- a/bitbake/lib/bb/daemonize.py
+++ b/bitbake/lib/bb/daemonize.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -74,26 +76,26 @@ def createDaemon(function, logfile):
     with open('/dev/null', 'r') as si:
         os.dup2(si.fileno(), sys.stdin.fileno())
 
-    try:
-        so = open(logfile, 'a+')
-        os.dup2(so.fileno(), sys.stdout.fileno())
-        os.dup2(so.fileno(), sys.stderr.fileno())
-    except io.UnsupportedOperation:
-        sys.stdout = open(logfile, 'a+')
+    with open(logfile, 'a+') as so:
+        try:
+            os.dup2(so.fileno(), sys.stdout.fileno())
+            os.dup2(so.fileno(), sys.stderr.fileno())
+        except io.UnsupportedOperation:
+            sys.stdout = so
 
-    # Have stdout and stderr be the same so log output matches chronologically
-    # and there aren't two seperate buffers
-    sys.stderr = sys.stdout
+        # Have stdout and stderr be the same so log output matches chronologically
+        # and there aren't two separate buffers
+        sys.stderr = sys.stdout
 
-    try:
-        function()
-    except Exception as e:
-        traceback.print_exc()
-    finally:
-        bb.event.print_ui_queue()
-        # os._exit() doesn't flush open files like os.exit() does. Manually flush
-        # stdout and stderr so that any logging output will be seen, particularly
-        # exception tracebacks.
-        sys.stdout.flush()
-        sys.stderr.flush()
-        os._exit(0)
+        try:
+            function()
+        except Exception as e:
+            traceback.print_exc()
+        finally:
+            bb.event.print_ui_queue()
+            # os._exit() doesn't flush open files like os.exit() does. Manually flush
+            # stdout and stderr so that any logging output will be seen, particularly
+            # exception tracebacks.
+            sys.stdout.flush()
+            sys.stderr.flush()
+            os._exit(0)
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 97022853ca..505f42950f 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -4,14 +4,16 @@ BitBake 'Data' implementations
 Functions for interacting with the data structure used by the
 BitBake build tools.
 
-The expandKeys and update_data are the most expensive
-operations. At night the cookie monster came by and
+expandKeys and datastore iteration are the most expensive
+operations. Updating overrides is now "on the fly" but still based
+on the idea of the cookie monster introduced by zecke:
+"At night the cookie monster came by and
 suggested 'give me cookies on setting the variables and
 things will work out'. Taking this suggestion into account
 applying the skills from the not yet passed 'Entwurf und
 Analyse von Algorithmen' lecture and the cookie
 monster seems to be right. We will track setVar more carefully
-to have faster update_data and expandKeys operations.
+to have faster datastore operations."
 
 This is a trade-off between speed and memory again but
 the speed is more critical here.
@@ -26,11 +28,6 @@ the speed is more critical here.
 
 import sys, os, re
 import hashlib
-if sys.argv[0][-5:] == "pydoc":
-    path = os.path.dirname(os.path.dirname(sys.argv[1]))
-else:
-    path = os.path.dirname(os.path.dirname(sys.argv[0]))
-sys.path.insert(0, path)
 from itertools import groupby
 
 from bb import data_smart
@@ -70,10 +67,6 @@ def keys(d):
     """Return a list of keys in d"""
     return d.keys()
 
-
-__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
-__expand_python_regexp__ = re.compile(r"\${@.+?}")
-
 def expand(s, d, varname = None):
     """Variable expansion using the data store"""
     return d.expand(s, varname)
@@ -121,8 +114,8 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     if d.getVarFlag(var, 'python', False) and func:
         return False
 
-    export = d.getVarFlag(var, "export", False)
-    unexport = d.getVarFlag(var, "unexport", False)
+    export = bb.utils.to_boolean(d.getVarFlag(var, "export"))
+    unexport = bb.utils.to_boolean(d.getVarFlag(var, "unexport"))
     if not all and not export and not unexport and not func:
         return False
 
@@ -195,8 +188,8 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
 
 def exported_keys(d):
     return (key for key in d.keys() if not key.startswith('__') and
-                                      d.getVarFlag(key, 'export', False) and
-                                      not d.getVarFlag(key, 'unexport', False))
+                                      bb.utils.to_boolean(d.getVarFlag(key, 'export')) and
+                                      not bb.utils.to_boolean(d.getVarFlag(key, 'unexport')))
 
 def exported_vars(d):
     k = list(exported_keys(d))
@@ -226,7 +219,7 @@ def emit_func(func, o=sys.__stdout__, d = init()):
         deps = newdeps
         seen |= deps
         newdeps = set()
-        for dep in deps:
+        for dep in sorted(deps):
             if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
                emit_var(dep, o, d, False) and o.write('\n')
                newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
@@ -268,65 +261,72 @@ def emit_func_python(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
-def update_data(d):
-    """Performs final steps upon the datastore, including application of overrides"""
-    d.finalize(parent = True)
+def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparsedata):
+    def handle_contains(value, contains, exclusions, d):
+        newvalue = []
+        if value:
+            newvalue.append(str(value))
+        for k in sorted(contains):
+            if k in exclusions or k in ignored_vars:
+                continue
+            l = (d.getVar(k) or "").split()
+            for item in sorted(contains[k]):
+                for word in item.split():
+                    if not word in l:
+                        newvalue.append("\n%s{%s} = Unset" % (k, item))
+                        break
+                else:
+                    newvalue.append("\n%s{%s} = Set" % (k, item))
+        return "".join(newvalue)
+
+    def handle_remove(value, deps, removes, d):
+        for r in sorted(removes):
+            r2 = d.expandWithRefs(r, None)
+            value += "\n_remove of %s" % r
+            deps |= r2.references
+            deps = deps | (keys & r2.execs)
+            value = handle_contains(value, r2.contains, exclusions, d)
+        return value
 
-def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     deps = set()
     try:
+        if key in mod_funcs:
+            exclusions = set()
+            moddep = bb.codeparser.modulecode_deps[key]
+            value = handle_contains("", moddep[3], exclusions, d)
+            return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value
+
         if key[-1] == ']':
             vf = key[:-1].split('[')
+            if vf[1] == "vardepvalueexclude":
+                return deps, ""
             value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            return deps, value
+            deps -= ignored_vars
+            return frozenset(deps), value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-
-        def handle_contains(value, contains, d):
-            newvalue = ""
-            for k in sorted(contains):
-                l = (d.getVar(k) or "").split()
-                for item in sorted(contains[k]):
-                    for word in item.split():
-                        if not word in l:
-                            newvalue += "\n%s{%s} = Unset" % (k, item)
-                            break
-                    else:
-                        newvalue += "\n%s{%s} = Set" % (k, item)
-            if not newvalue:
-                return value
-            if not value:
-                return newvalue
-            return value + newvalue
-
-        def handle_remove(value, deps, removes, d):
-            for r in sorted(removes):
-                r2 = d.expandWithRefs(r, None)
-                value += "\n_remove of %s" % r
-                deps |= r2.references
-                deps = deps | (keys & r2.execs)
-            return value
+        exclusions = varflags.get("vardepsexclude", "").split()
 
         if "vardepvalue" in varflags:
             value = varflags.get("vardepvalue")
         elif varflags.get("func"):
             if varflags.get("python"):
-                value = d.getVarFlag(key, "_content", False)
+                value = codeparsedata.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
-                value = handle_contains(value, parser.contains, d)
+                value = handle_contains(value, parser.contains, exclusions, d)
             else:
-                value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
+                value, parsedvar = codeparsedata.getVarFlag(key, "_content", False, retparser=True)
                 parser = bb.codeparser.ShellParser(key, logger)
                 parser.parse_shell(parsedvar.value)
                 deps = deps | shelldeps
                 deps = deps | parsedvar.references
                 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
-                value = handle_contains(value, parsedvar.contains, d)
+                value = handle_contains(value, parsedvar.contains, exclusions, d)
                 if hasattr(parsedvar, "removes"):
                     value = handle_remove(value, deps, parsedvar.removes, d)
             if vardeps is None:
@@ -341,7 +341,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             value, parser = d.getVarFlag(key, "_content", False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            value = handle_contains(value, parser.contains, d)
+            value = handle_contains(value, parser.contains, exclusions, d)
             if hasattr(parser, "removes"):
                 value = handle_remove(value, deps, parser.removes, d)
 
@@ -361,43 +361,50 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 deps |= set(varfdeps)
 
         deps |= set((vardeps or "").split())
-        deps -= set(varflags.get("vardepsexclude", "").split())
+        deps -= set(exclusions)
+        deps -= ignored_vars
     except bb.parse.SkipRecipe:
         raise
     except Exception as e:
         bb.warn("Exception during build_dependencies for %s" % key)
         raise
-    return deps, value
+    return frozenset(deps), value
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d, whitelist):
+def generate_dependencies(d, ignored_vars):
 
-    keys = set(key for key in d if not key.startswith("__"))
-    shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
+    mod_funcs = set(bb.codeparser.modulecode_deps.keys())
+    keys = set(key for key in d if not key.startswith("__")) | mod_funcs
+    shelldeps = set(key for key in d.getVar("__exportlist", False) if bb.utils.to_boolean(d.getVarFlag(key, "export")) and not bb.utils.to_boolean(d.getVarFlag(key, "unexport")))
     varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
 
+    codeparserd = d.createCopy()
+    for forced in (d.getVar('BB_HASH_CODEPARSER_VALS') or "").split():
+        key, value = forced.split("=", 1)
+        codeparserd.setVar(key, value)
+
     deps = {}
     values = {}
 
     tasklist = d.getVar('__BBTASKS', False) or []
     for task in tasklist:
-        deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
+        deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps - whitelist
+            nextdeps = newdeps
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
                 if dep not in deps:
-                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
+                    deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
                 newdeps |=  deps[dep]
             newdeps -= seen
         #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
-def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
+def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
     taskdeps = {}
     basehash = {}
 
@@ -406,9 +413,10 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
 
         if data is None:
             bb.error("Task %s from %s seems to be empty?!" % (task, fn))
-            data = ''
+            data = []
+        else:
+            data = [data]
 
-        gendeps[task] -= whitelist
         newdeps = gendeps[task]
         seen = set()
         while newdeps:
@@ -416,27 +424,24 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
-                if dep in whitelist:
-                    continue
-                gendeps[dep] -= whitelist
                 newdeps |= gendeps[dep]
             newdeps -= seen
 
         alldeps = sorted(seen)
         for dep in alldeps:
-            data = data + dep
+            data.append(dep)
             var = lookupcache[dep]
             if var is not None:
-                data = data + str(var)
+                data.append(str(var))
         k = fn + ":" + task
-        basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
-        taskdeps[task] = alldeps
+        basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest()
+        taskdeps[task] = frozenset(seen)
 
     return taskdeps, basehash
 
 def inherits_class(klass, d):
     val = d.getVar('__inherit_cache', False) or []
-    needle = os.path.join('classes', '%s.bbclass' % klass)
+    needle = '/%s.bbclass' % klass
     for v in val:
         if v.endswith(needle):
             return True
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 2328c334ac..0128a5bb17 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -16,8 +16,11 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import copy, re, sys, traceback
-from collections import MutableMapping
+import builtins
+import copy
+import re
+import sys
+from collections.abc import MutableMapping
 import logging
 import hashlib
 import bb, bb.codeparser
@@ -26,13 +29,25 @@ from bb.COW  import COWDictBase
 
 logger = logging.getLogger("BitBake.Data")
 
-__setvar_keyword__ = ["_append", "_prepend", "_remove"]
-__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
-__expand_python_regexp__ = re.compile(r"\${@.+?}")
+__setvar_keyword__ = [":append", ":prepend", ":remove"]
+__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
+__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
 
+bitbake_renamed_vars = {
+    "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
+    "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
+    "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
+    "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
+    "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
+    "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
+    "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
+    "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
+    "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
+}
+
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
     # Save effort.
@@ -80,10 +95,11 @@ def infer_caller_details(loginfo, parent = False, varval = True):
             loginfo['func'] = func
 
 class VariableParse:
-    def __init__(self, varname, d, val = None):
+    def __init__(self, varname, d, unexpanded_value = None, val = None):
         self.varname = varname
         self.d = d
         self.value = val
+        self.unexpanded_value = unexpanded_value
 
         self.references = set()
         self.execs = set()
@@ -107,6 +123,11 @@ class VariableParse:
             else:
                 code = match.group()[3:-1]
 
+            # Do not run code that contains one or more unexpanded variables
+            # instead return the code with the characters we removed put back
+            if __expand_var_regexp__.findall(code):
+                return "${@" + code + "}"
+
             if self.varname:
                 varname = 'Var <%s>' % self.varname
             else:
@@ -132,16 +153,21 @@ class VariableParse:
             value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
             return str(value)
 
-
 class DataContext(dict):
+    excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
+
     def __init__(self, metadata, **kwargs):
         self.metadata = metadata
         dict.__init__(self, **kwargs)
         self['d'] = metadata
+        self.context = set(bb.utils.get_context())
 
     def __missing__(self, key):
+        if key in self.excluded or key in self.context:
+            raise KeyError(key)
+
         value = self.metadata.getVar(key)
-        if value is None or self.metadata.getVarFlag(key, 'func', False):
+        if value is None:
             raise KeyError(key)
         else:
             return value
@@ -151,6 +177,7 @@ class ExpansionError(Exception):
         self.expression = expression
         self.variablename = varname
         self.exception = exception
+        self.varlist = [varname or expression or ""]
         if varname:
             if expression:
                 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
@@ -160,8 +187,14 @@ class ExpansionError(Exception):
             self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
         Exception.__init__(self, self.msg)
         self.args = (varname, expression, exception)
+
+    def addVar(self, varname):
+        if varname:
+            self.varlist.append(varname)
+
     def __str__(self):
-        return self.msg
+        chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
+        return self.msg + chain
 
 class IncludeHistory(object):
     def __init__(self, parent = None, filename = '[TOP LEVEL]'):
@@ -277,7 +310,7 @@ class VariableHistory(object):
             for (r, override) in d.overridedata[var]:
                 for event in self.variable(r):
                     loginfo = event.copy()
-                    if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
+                    if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
                         continue
                     loginfo['variable'] = var
                     loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
@@ -329,6 +362,16 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
+    def get_variable_refs(self, var):
+        """Return a dict of file/line references"""
+        var_history = self.variable(var)
+        refs = {}
+        for event in var_history:
+            if event['file'] not in refs:
+                refs[event['file']] = []
+            refs[event['file']].append(event['line'])
+        return refs
+
     def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
@@ -342,7 +385,7 @@ class VariableHistory(object):
         for event in history:
             if 'flag' in event:
                 continue
-            if event['op'] == '_remove':
+            if event['op'] == ':remove':
                 continue
             if isset and event['op'] == 'set?':
                 continue
@@ -363,6 +406,23 @@ class VariableHistory(object):
             else:
                 self.variables[var] = []
 
+def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
+    info = ""
+    if "file" in loginfo:
+        info = " file: %s" % loginfo["file"]
+    if "line" in loginfo:
+        info += " line: %s" % loginfo["line"]
+    if fullvar and fullvar != var:
+        info += " referenced as: %s" % fullvar
+    if info:
+        info = " (%s)" % info.strip()
+    renameinfo = renamedvars[var]
+    if " " in renameinfo:
+        # A space signals a string to display instead of a rename
+        bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
+    else:
+        bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
+
 class DataSmart(MutableMapping):
     def __init__(self):
         self.dict = {}
@@ -370,6 +430,8 @@ class DataSmart(MutableMapping):
         self.inchistory = IncludeHistory()
         self.varhistory = VariableHistory(self)
         self._tracking = False
+        self._var_renames = {}
+        self._var_renames.update(bitbake_renamed_vars)
 
         self.expand_cache = {}
 
@@ -391,9 +453,9 @@ class DataSmart(MutableMapping):
     def expandWithRefs(self, s, varname):
 
         if not isinstance(s, str): # sanity check
-            return VariableParse(varname, self, s)
+            return VariableParse(varname, self, s, s)
 
-        varparse = VariableParse(varname, self)
+        varparse = VariableParse(varname, self, s)
 
         while s.find('${') != -1:
             olds = s
@@ -403,14 +465,17 @@ class DataSmart(MutableMapping):
                     s = __expand_python_regexp__.sub(varparse.python_sub, s)
                 except SyntaxError as e:
                     # Likely unmatched brackets, just don't expand the expression
-                    if e.msg != "EOL while scanning string literal":
+                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
                         raise
                 if s == olds:
                     break
-            except ExpansionError:
+            except ExpansionError as e:
+                e.addVar(varname)
                 raise
             except bb.parse.SkipRecipe:
                 raise
+            except bb.BBHandledException:
+                raise
             except Exception as exc:
                 tb = sys.exc_info()[2]
                 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
@@ -422,24 +487,19 @@ class DataSmart(MutableMapping):
     def expand(self, s, varname = None):
         return self.expandWithRefs(s, varname).value
 
-    def finalize(self, parent = False):
-        return
-
-    def internal_finalize(self, parent = False):
-        """Performs final steps upon the datastore, including application of overrides"""
-        self.overrides = None
-
     def need_overrides(self):
         if self.overrides is not None:
             return
         if self.inoverride:
             return
+        overrride_stack = []
         for count in range(5):
             self.inoverride = True
             # Can end up here recursively so setup dummy values
             self.overrides = []
             self.overridesset = set()
             self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
+            overrride_stack.append(self.overrides)
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
@@ -449,7 +509,7 @@ class DataSmart(MutableMapping):
             self.overrides = newoverrides
             self.overridesset = set(self.overrides)
         else:
-            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
+            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack))
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -460,27 +520,44 @@ class DataSmart(MutableMapping):
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var], self.overridedata.get(var, None)
+                return dest[var]
 
             if "_data" not in dest:
                 break
             dest = dest["_data"]
-        return None, self.overridedata.get(var, None)
+        return None
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
         else:
             self.initVar(var)
 
+    def hasOverrides(self, var):
+        return var in self.overridedata
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+
+        if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
+            info = "%s" % var
+            if "file" in loginfo:
+                info += " file: %s" % loginfo["file"]
+            if "line" in loginfo:
+                info += " line: %s" % loginfo["line"]
+            bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
+
+        shortvar = var.split(":", 1)[0]
+        if shortvar in self._var_renames:
+            _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
+            # Mark that we have seen a renamed variable
+            self.setVar("_FAILPARSINGERRORHANDLED", True)
+
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -509,7 +586,7 @@ class DataSmart(MutableMapping):
             # pay the cookie monster
 
             # more cookies for the cookie monster
-            if '_' in var:
+            if ':' in var:
                 self._setvar_update_overrides(base, **loginfo)
 
             if base in self.overridevars:
@@ -520,27 +597,27 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
 
         if not parsing:
-            if "_append" in self.dict[var]:
-                del self.dict[var]["_append"]
-            if "_prepend" in self.dict[var]:
-                del self.dict[var]["_prepend"]
-            if "_remove" in self.dict[var]:
-                del self.dict[var]["_remove"]
+            if ":append" in self.dict[var]:
+                del self.dict[var][":append"]
+            if ":prepend" in self.dict[var]:
+                del self.dict[var][":prepend"]
+            if ":remove" in self.dict[var]:
+                del self.dict[var][":remove"]
             if var in self.overridedata:
                 active = []
                 self.need_overrides()
                 for (r, o) in self.overridedata[var]:
                     if o in self.overridesset:
                         active.append(r)
-                    elif "_" in o:
-                        if set(o.split("_")).issubset(self.overridesset):
+                    elif ":" in o:
+                        if set(o.split(":")).issubset(self.overridesset):
                             active.append(r)
                 for a in active:
                     self.delVar(a)
                 del self.overridedata[var]
 
         # more cookies for the cookie monster
-        if '_' in var:
+        if ':' in var:
             self._setvar_update_overrides(var, **loginfo)
 
         # setting var
@@ -562,12 +639,12 @@ class DataSmart(MutableMapping):
                 nextnew.update(vardata.references)
                 nextnew.update(vardata.contains.keys())
             new = nextnew
-        self.internal_finalize(True)
+        self.overrides = None
 
     def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
-        override = var[var.rfind('_')+1:]
-        shortvar = var[:var.rfind('_')]
+        override = var[var.rfind(':')+1:]
+        shortvar = var[:var.rfind(':')]
         while override and __override_regexp__.match(override):
             if shortvar not in self.overridedata:
                 self.overridedata[shortvar] = []
@@ -576,9 +653,9 @@ class DataSmart(MutableMapping):
                 self.overridedata[shortvar] = list(self.overridedata[shortvar])
                 self.overridedata[shortvar].append([var, override])
             override = None
-            if "_" in shortvar:
-                override = var[shortvar.rfind('_')+1:]
-                shortvar = var[:shortvar.rfind('_')]
+            if ":" in shortvar:
+                override = var[shortvar.rfind(':')+1:]
+                shortvar = var[:shortvar.rfind(':')]
                 if len(shortvar) == 0:
                     override = None
 
@@ -602,10 +679,11 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
             self.setVar(newkey, val, ignore=True, parsing=True)
 
-        for i in (__setvar_keyword__):
-            src = self.getVarFlag(key, i, False)
-            if src is None:
+        srcflags = self.getVarFlags(key, False, True) or {}
+        for i in srcflags:
+            if i not in (__setvar_keyword__):
                 continue
+            src = srcflags[i]
 
             dest = self.getVarFlag(newkey, i, False) or []
             dest.extend(src)
@@ -617,7 +695,7 @@ class DataSmart(MutableMapping):
                 self.overridedata[newkey].append([v.replace(key, newkey), o])
                 self.renameVar(v, v.replace(key, newkey))
 
-        if '_' in newkey and val is None:
+        if ':' in newkey and val is None:
             self._setvar_update_overrides(newkey, **loginfo)
 
         loginfo['variable'] = key
@@ -629,12 +707,12 @@ class DataSmart(MutableMapping):
     def appendVar(self, var, value, **loginfo):
         loginfo['op'] = 'append'
         self.varhistory.record(**loginfo)
-        self.setVar(var + "_append", value, ignore=True, parsing=True)
+        self.setVar(var + ":append", value, ignore=True, parsing=True)
 
     def prependVar(self, var, value, **loginfo):
         loginfo['op'] = 'prepend'
         self.varhistory.record(**loginfo)
-        self.setVar(var + "_prepend", value, ignore=True, parsing=True)
+        self.setVar(var + ":prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
         self.expand_cache = {}
@@ -645,10 +723,10 @@ class DataSmart(MutableMapping):
         self.dict[var] = {}
         if var in self.overridedata:
             del self.overridedata[var]
-        if '_' in var:
-            override = var[var.rfind('_')+1:]
-            shortvar = var[:var.rfind('_')]
-            while override and override.islower():
+        if ':' in var:
+            override = var[var.rfind(':')+1:]
+            shortvar = var[:var.rfind(':')]
+            while override and __override_regexp__.match(override):
                 try:
                     if shortvar in self.overridedata:
                         # Force CoW by recreating the list first
@@ -657,15 +735,23 @@ class DataSmart(MutableMapping):
                 except ValueError as e:
                     pass
                 override = None
-                if "_" in shortvar:
-                    override = var[shortvar.rfind('_')+1:]
-                    shortvar = var[:shortvar.rfind('_')]
+                if ":" in shortvar:
+                    override = var[shortvar.rfind(':')+1:]
+                    shortvar = var[:shortvar.rfind(':')]
                     if len(shortvar) == 0:
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
 
+        if var == "BB_RENAMED_VARIABLES":
+            self._var_renames[flag] = value
+
+        if var in self._var_renames:
+            _print_rename_error(var, loginfo, self._var_renames)
+            # Mark that we have seen a renamed variable
+            self.setVar("_FAILPARSINGERRORHANDLED", True)
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -674,7 +760,7 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
         self.dict[var][flag] = value
 
-        if flag == "_defaultval" and '_' in var:
+        if flag == "_defaultval" and ':' in var:
             self._setvar_update_overrides(var, **loginfo)
         if flag == "_defaultval" and var in self.overridevars:
             self._setvar_update_overridevars(var, value)
@@ -695,22 +781,27 @@ class DataSmart(MutableMapping):
                 return None
             cachename = var + "[" + flag + "]"
 
+        if not expand and retparser and cachename in self.expand_cache:
+            return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename]
+
         if expand and cachename in self.expand_cache:
             return self.expand_cache[cachename].value
 
-        local_var, overridedata = self._findVar(var)
+        local_var = self._findVar(var)
         value = None
         removes = set()
-        if flag == "_content" and overridedata is not None and not parsing:
+        if flag == "_content" and not parsing:
+            overridedata = self.overridedata.get(var, None)
+        if flag == "_content" and not parsing and overridedata is not None:
             match = False
             active = {}
             self.need_overrides()
             for (r, o) in overridedata:
-                # What about double overrides both with "_" in the name?
+                # FIXME What about double overrides both with "_" in the name?
                 if o in self.overridesset:
                     active[o] = r
-                elif "_" in o:
-                    if set(o.split("_")).issubset(self.overridesset):
+                elif ":" in o:
+                    if set(o.split(":")).issubset(self.overridesset):
                         active[o] = r
 
             mod = True
@@ -718,10 +809,10 @@ class DataSmart(MutableMapping):
                 mod = False
                 for o in self.overrides:
                     for a in active.copy():
-                        if a.endswith("_" + o):
+                        if a.endswith(":" + o):
                             t = active[a]
                             del active[a]
-                            active[a.replace("_" + o, "")] = t
+                            active[a.replace(":" + o, "")] = t
                             mod = True
                         elif a == o:
                             match = active[a]
@@ -740,31 +831,31 @@ class DataSmart(MutableMapping):
                 value = copy.copy(local_var["_defaultval"])
 
 
-        if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
-            if not value:
-                value = ""
+        if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_append"]:
+            for (r, o) in local_var[":append"]:
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
+                    if value is None:
+                        value = ""
                     value = value + r
 
-        if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
-            if not value:
-                value = ""
+        if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_prepend"]:
+            for (r, o) in local_var[":prepend"]:
 
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
+                    if value is None:
+                        value = ""
                     value = r + value
 
         parser = None
@@ -773,12 +864,12 @@ class DataSmart(MutableMapping):
         if expand:
             value = parser.value
 
-        if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
+        if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_remove"]:
+            for (r, o) in local_var[":remove"]:
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
@@ -791,7 +882,7 @@ class DataSmart(MutableMapping):
                     expanded_removes[r] = self.expand(r).split()
 
                 parser.removes = set()
-                val = ""
+                val = []
                 for v in __whitespace_split__.split(parser.value):
                     skip = False
                     for r in removes:
@@ -800,8 +891,8 @@ class DataSmart(MutableMapping):
                             skip = True
                     if skip:
                         continue
-                    val = val + v
-                parser.value = val
+                    val.append(v)
+                parser.value = "".join(val)
                 if expand:
                     value = parser.value
 
@@ -816,7 +907,7 @@ class DataSmart(MutableMapping):
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
 
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -859,12 +950,12 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
         flags = {}
 
         if local_var:
             for i in local_var:
-                if i.startswith("_") and not internalflags:
+                if i.startswith(("_", ":")) and not internalflags:
                     continue
                 flags[i] = local_var[i]
                 if expand and i in expand:
@@ -905,6 +996,7 @@ class DataSmart(MutableMapping):
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
+        data._var_renames = self._var_renames
 
         data.overrides = None
         data.overridevars = copy.copy(self.overridevars)
@@ -927,7 +1019,7 @@ class DataSmart(MutableMapping):
         value = self.getVar(variable, False)
         for key in keys:
             referrervalue = self.getVar(key, False)
-            if referrervalue and ref in referrervalue:
+            if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
                 self.setVar(key, referrervalue.replace(ref, value))
 
     def localkeys(self):
@@ -962,8 +1054,8 @@ class DataSmart(MutableMapping):
             for (r, o) in self.overridedata[var]:
                 if o in self.overridesset:
                     overrides.add(var)
-                elif "_" in o:
-                    if set(o.split("_")).issubset(self.overridesset):
+                elif ":" in o:
+                    if set(o.split(":")).issubset(self.overridesset):
                         overrides.add(var)
 
         for k in keylist(self.dict):
@@ -993,10 +1085,10 @@ class DataSmart(MutableMapping):
         d = self.createCopy()
         bb.data.expandKeys(d)
 
-        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
+        config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))
         for key in keys:
-            if key in config_whitelist:
+            if key in config_ignore_vars:
                 continue
 
             value = d.getVar(key, False) or ""
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 23e1f3187b..4761c86880 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -40,7 +40,7 @@ class HeartbeatEvent(Event):
     """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
        (runQueueTaskStarted when there are many short tasks) or not at all for long periods
        of time (again runQueueTaskStarted, when there is just one long-running task), so this
-       event is more suitable for doing some task-independent work occassionally."""
+       event is more suitable for doing some task-independent work occasionally."""
     def __init__(self, time):
         Event.__init__(self)
         self.time = time
@@ -68,29 +68,39 @@ _catchall_handlers = {}
 _eventfilter = None
 _uiready = False
 _thread_lock = threading.Lock()
-_thread_lock_enabled = False
-
-if hasattr(__builtins__, '__setitem__'):
-    builtins = __builtins__
-else:
-    builtins = __builtins__.__dict__
+_heartbeat_enabled = False
+_should_exit = threading.Event()
 
 def enable_threadlock():
-    global _thread_lock_enabled
-    _thread_lock_enabled = True
+    # Always needed now
+    return
 
 def disable_threadlock():
-    global _thread_lock_enabled
-    _thread_lock_enabled = False
+    # Always needed now
+    return
+
+def enable_heartbeat():
+    global _heartbeat_enabled
+    _heartbeat_enabled = True
+
+def disable_heartbeat():
+    global _heartbeat_enabled
+    _heartbeat_enabled = False
+
+#
+# In long running code, this function should be called periodically
+# to check if we should exit due to an interuption (.e.g Ctrl+C from the UI)
+#
+def check_for_interrupts(d):
+    global _should_exit
+    if _should_exit.is_set():
+        bb.warn("Exiting due to interrupt.")
+        raise bb.BBHandledException()
 
 def execute_handler(name, handler, event, d):
     event.data = d
-    addedd = False
-    if 'd' not in builtins:
-        builtins['d'] = d
-        addedd = True
     try:
-        ret = handler(event)
+        ret = handler(event, d)
     except (bb.parse.SkipRecipe, bb.BBHandledException):
         raise
     except Exception:
@@ -104,8 +114,7 @@ def execute_handler(name, handler, event, d):
         raise
     finally:
         del event.data
-        if addedd:
-            del builtins['d']
+
 
 def fire_class_handlers(event, d):
     if isinstance(event, logging.LogRecord):
@@ -118,7 +127,7 @@ def fire_class_handlers(event, d):
             if _eventfilter:
                 if not _eventfilter(name, handler, event, d):
                     continue
-            if d and not name in (d.getVar("__BBHANDLERS_MC") or []):
+            if d is not None and not name in (d.getVar("__BBHANDLERS_MC") or set()):
                 continue
             execute_handler(name, handler, event, d)
 
@@ -132,8 +141,14 @@ def print_ui_queue():
     if not _uiready:
         from bb.msg import BBLogFormatter
         # Flush any existing buffered content
-        sys.stdout.flush()
-        sys.stderr.flush()
+        try:
+            sys.stdout.flush()
+        except:
+            pass
+        try:
+            sys.stderr.flush()
+        except:
+            pass
         stdout = logging.StreamHandler(sys.stdout)
         stderr = logging.StreamHandler(sys.stderr)
         formatter = BBLogFormatter("%(levelname)s: %(message)s")
@@ -174,36 +189,30 @@ def print_ui_queue():
 
 def fire_ui_handlers(event, d):
     global _thread_lock
-    global _thread_lock_enabled
 
     if not _uiready:
         # No UI handlers registered yet, queue up the messages
         ui_queue.append(event)
         return
 
-    if _thread_lock_enabled:
-        _thread_lock.acquire()
-
-    errors = []
-    for h in _ui_handlers:
-        #print "Sending event %s" % event
-        try:
-             if not _ui_logfilters[h].filter(event):
-                 continue
-             # We use pickle here since it better handles object instances
-             # which xmlrpc's marshaller does not. Events *must* be serializable
-             # by pickle.
-             if hasattr(_ui_handlers[h].event, "sendpickle"):
-                _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
-             else:
-                _ui_handlers[h].event.send(event)
-        except:
-            errors.append(h)
-    for h in errors:
-        del _ui_handlers[h]
-
-    if _thread_lock_enabled:
-        _thread_lock.release()
+    with bb.utils.lock_timeout(_thread_lock):
+        errors = []
+        for h in _ui_handlers:
+            #print "Sending event %s" % event
+            try:
+                 if not _ui_logfilters[h].filter(event):
+                     continue
+                 # We use pickle here since it better handles object instances
+                 # which xmlrpc's marshaller does not. Events *must* be serializable
+                 # by pickle.
+                 if hasattr(_ui_handlers[h].event, "sendpickle"):
+                    _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+                 else:
+                    _ui_handlers[h].event.send(event)
+            except:
+                errors.append(h)
+        for h in errors:
+            del _ui_handlers[h]
 
 def fire(event, d):
     """Fire off an Event"""
@@ -232,26 +241,31 @@ noop = lambda _: None
 def register(name, handler, mask=None, filename=None, lineno=None, data=None):
     """Register an Event handler"""
 
-    if data and data.getVar("BB_CURRENT_MC"):
+    if data is not None and data.getVar("BB_CURRENT_MC"):
         mc = data.getVar("BB_CURRENT_MC")
         name = '%s%s' % (mc.replace('-', '_'), name)
 
     # already registered
     if name in _handlers:
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
         return AlreadyRegistered
 
     if handler is not None:
         # handle string containing python code
         if isinstance(handler, str):
-            tmp = "def %s(e):\n%s" % (name, handler)
+            tmp = "def %s(e, d):\n%s" % (name, handler)
+            # Inject empty lines to make code match lineno in filename
+            if lineno is not None:
+                tmp = "\n" * (lineno-1) + tmp
             try:
                 code = bb.methodpool.compile_cache(tmp)
                 if not code:
                     if filename is None:
-                        filename = "%s(e)" % name
+                        filename = "%s(e, d)" % name
                     code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
-                    if lineno is not None:
-                        ast.increment_lineno(code, lineno-1)
                     code = compile(code, filename, "exec")
                     bb.methodpool.compile_cache_add(tmp, code)
             except SyntaxError:
@@ -274,16 +288,16 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None):
                     _event_handler_map[m] = {}
                 _event_handler_map[m][name] = True
 
-        if data:
-            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
-            bbhands_mc.append(name)
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
             data.setVar("__BBHANDLERS_MC", bbhands_mc)
 
         return Registered
 
 def remove(name, handler, data=None):
     """Remove an Event handler"""
-    if data:
+    if data is not None:
         if data.getVar("BB_CURRENT_MC"):
             mc = data.getVar("BB_CURRENT_MC")
             name = '%s%s' % (mc.replace('-', '_'), name)
@@ -295,8 +309,8 @@ def remove(name, handler, data=None):
         if name in _event_handler_map[event]:
             _event_handler_map[event].pop(name)
 
-    if data:
-        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
+    if data is not None:
+        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
         if name in bbhands_mc:
             bbhands_mc.remove(name)
             data.setVar("__BBHANDLERS_MC", bbhands_mc)
@@ -313,21 +327,23 @@ def set_eventfilter(func):
     _eventfilter = func
 
 def register_UIHhandler(handler, mainui=False):
-    bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
-    _ui_handlers[_ui_handler_seq] = handler
-    level, debug_domains = bb.msg.constructLogOptions()
-    _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
-    if mainui:
-        global _uiready
-        _uiready = _ui_handler_seq
-    return _ui_handler_seq
+    with bb.utils.lock_timeout(_thread_lock):
+        bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
+        _ui_handlers[_ui_handler_seq] = handler
+        level, debug_domains = bb.msg.constructLogOptions()
+        _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
+        if mainui:
+            global _uiready
+            _uiready = _ui_handler_seq
+        return _ui_handler_seq
 
 def unregister_UIHhandler(handlerNum, mainui=False):
     if mainui:
         global _uiready
         _uiready = False
-    if handlerNum in _ui_handlers:
-        del _ui_handlers[handlerNum]
+    with bb.utils.lock_timeout(_thread_lock):
+        if handlerNum in _ui_handlers:
+            del _ui_handlers[handlerNum]
     return
 
 def get_uihandler():
@@ -482,7 +498,7 @@ class BuildCompleted(BuildBase, OperationCompleted):
         BuildBase.__init__(self, n, p, failures)
 
 class DiskFull(Event):
-    """Disk full case build aborted"""
+    """Disk full case build halted"""
     def __init__(self, dev, type, freespace, mountpoint):
         Event.__init__(self)
         self._dev = dev
@@ -666,6 +682,17 @@ class ReachableStamps(Event):
         Event.__init__(self)
         self.stamps = stamps
 
+class StaleSetSceneTasks(Event):
+    """
+    An event listing setscene tasks which are 'stale' and will
+    be rerun. The metadata may use to clean up stale data.
+    tasks is a mapping of tasks and matching stale stamps.
+    """
+
+    def __init__(self, tasks):
+        Event.__init__(self)
+        self.tasks = tasks
+
 class FilesMatchingFound(Event):
     """
     Event when a list of files matching the supplied pattern has
@@ -749,7 +776,7 @@ class LogHandler(logging.Handler):
 class MetadataEvent(Event):
     """
     Generic event that target for OE-Core classes
-    to report information during asynchrous execution
+    to report information during asynchronous execution
     """
     def __init__(self, eventtype, eventdata):
         Event.__init__(self)
@@ -830,3 +857,19 @@ class FindSigInfoResult(Event):
     def __init__(self, result):
         Event.__init__(self)
         self.result = result
+
+class GetTaskSignatureResult(Event):
+    """
+    Event to return results from GetTaskSignatures command
+    """
+    def __init__(self, sig):
+        Event.__init__(self)
+        self.sig = sig
+
+class ParseError(Event):
+    """
+    Event to indicate parse failed
+    """
+    def __init__(self, msg):
+        super().__init__()
+        self._msg = msg
diff --git a/bitbake/lib/bb/exceptions.py b/bitbake/lib/bb/exceptions.py
index ecbad59970..801db9c82f 100644
--- a/bitbake/lib/bb/exceptions.py
+++ b/bitbake/lib/bb/exceptions.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README
new file mode 100644
index 0000000000..67b787ef47
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/README
@@ -0,0 +1,57 @@
+There are expectations of users of the fetcher code. This file attempts to document
+some of the constraints that are present. Some are obvious, some are less so. It is
+documented in the context of how OE uses it but the API calls are generic.
+
+a) network access for sources is only expected to happen in the do_fetch task.
+   This is not enforced or tested but is required so that we can:
+
+   i) audit the sources used (i.e. for license/manifest reasons) 
+   ii) support offline builds with a suitable cache
+   iii) allow work to continue even with downtime upstream
+   iv) allow for changes upstream in incompatible ways
+   v) allow rebuilding of the software in X years time
+
+b) network access is not expected in do_unpack task.
+
+c) you can take DL_DIR and use it as a mirror for offline builds.
+
+d) access to the network is only made when explicitly configured in recipes
+   (e.g. use of AUTOREV, or use of git tags which change revision).
+
+e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it 
+   will match in future exactly in a clean build with a new DL_DIR).
+   One specific pain point example are git tags. They can be replaced and change
+   so the git fetcher has to resolve them with the network. We use git revisions
+   where possible to avoid this and ensure determinism.
+
+f) network access is expected to work with the standard linux proxy variables
+   so that access behind firewalls works (the fetcher sets these in the 
+   environment but only in the do_fetch tasks).
+
+g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV 
+   git recipe might be ok but you can't expect to checkout a git tree.
+
+h) we need to provide revision information during parsing such that a version
+   for the recipe can be constructed.
+
+i) versions are expected to be able to increase in a way which sorts allowing 
+   package feeds to operate (see PR server required for git revisions to sort).
+
+j) API to query for possible version upgrades of a url is highly desireable to 
+   allow our automated upgrage code to function (it is implied this does always 
+   have network access).
+
+k) Where fixes or changes to behaviour in the fetcher are made, we ask that 
+   test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do 
+   have fairly extensive test coverage of the fetcher as it is the only way
+   to track all of its corner cases, it still doesn't give entire coverage 
+   though sadly.
+   
+l) If using tools during parse time, they will have to be in ASSUME_PROVIDED
+   in OE's context as we can't build git-native, then parse a recipe and use
+   git ls-remote.
+
+Not all fetchers support all features, autorev is optional and doesn't make
+sense for some. Upgrade detection means different things in different contexts
+too.
+
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 19169d780f..5bf2c4b8cf 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -113,7 +113,7 @@ class MissingParameterError(BBFetchException):
         self.args = (missing, url)
 
 class ParameterError(BBFetchException):
-    """Exception raised when a url cannot be proccessed due to invalid parameters."""
+    """Exception raised when a url cannot be processed due to invalid parameters."""
     def __init__(self, message, url):
         msg = "URL: '%s' has invalid parameters. %s" % (url, message)
         self.url = url
@@ -182,7 +182,7 @@ class URI(object):
     Some notes about relative URIs: while it's specified that
     a URI beginning with <scheme>:// should either be directly
     followed by a hostname or a /, the old URI handling of the
-    fetch2 library did not comform to this. Therefore, this URI
+    fetch2 library did not conform to this. Therefore, this URI
     class has some kludges to make sure that URIs are parsed in
     a way comforming to bitbake's current usage. This URI class
     supports the following:
@@ -199,7 +199,7 @@ class URI(object):
      file://hostname/absolute/path.diff (would be IETF compliant)
 
     Note that the last case only applies to a list of
-    "whitelisted" schemes (currently only file://), that requires
+    explicitly allowed schemes (currently only file://), that requires
     its URIs to not have a network location.
     """
 
@@ -290,12 +290,12 @@ class URI(object):
 
     def _param_str_split(self, string, elmdelim, kvdelim="="):
         ret = collections.OrderedDict()
-        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
+        for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
             ret[k] = v
         return ret
 
     def _param_str_join(self, dict_, elmdelim, kvdelim="="):
-        return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()])
+        return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
 
     @property
     def hostport(self):
@@ -388,7 +388,7 @@ def decodeurl(url):
             if s:
                 if not '=' in s:
                     raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
-                s1, s2 = s.split('=')
+                s1, s2 = s.split('=', 1)
                 p[s1] = s2
 
     return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -402,24 +402,24 @@ def encodeurl(decoded):
 
     if not type:
         raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
-    url = '%s://' % type
+    url = ['%s://' % type]
     if user and type != "file":
-        url += "%s" % user
+        url.append("%s" % user)
         if pswd:
-            url += ":%s" % pswd
-        url += "@"
+            url.append(":%s" % pswd)
+        url.append("@")
     if host and type != "file":
-        url += "%s" % host
+        url.append("%s" % host)
     if path:
         # Standardise path to ensure comparisons work
         while '//' in path:
             path = path.replace("//", "/")
-        url += "%s" % urllib.parse.quote(path)
+        url.append("%s" % urllib.parse.quote(path))
     if p:
         for parm in p:
-            url += ";%s=%s" % (parm, p[parm])
+            url.append(";%s=%s" % (parm, p[parm]))
 
-    return url
+    return "".join(url)
 
 def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     if not ud.url or not uri_find or not uri_replace:
@@ -430,6 +430,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     uri_replace_decoded = list(decodeurl(uri_replace))
     logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
+    # 0 - type, 1 - host, 2 - path, 3 - user,  4- pswd, 5 - params
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
         regexp = i
@@ -449,6 +450,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 for l in replacements:
                     uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
                 result_decoded[loc][k] = uri_replace_decoded[loc][k]
+        elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
+            # User/password in the replacement is just a straight replacement
+            result_decoded[loc] = uri_replace_decoded[loc]
         elif (re.match(regexp, uri_decoded[loc])):
             if not uri_replace_decoded[loc]:
                 result_decoded[loc] = ""
@@ -465,10 +469,18 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                     basename = os.path.basename(mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
+                    uri_find_decoded[5] = {}
                 elif ud.localpath and ud.method.supports_checksum(ud):
                     basename = os.path.basename(ud.localpath)
-                if basename and not result_decoded[loc].endswith(basename):
-                    result_decoded[loc] = os.path.join(result_decoded[loc], basename)
+                if basename:
+                    uri_basename = os.path.basename(uri_decoded[loc])
+                    # Prefix with a slash as a sentinel in case
+                    # result_decoded[loc] does not contain one.
+                    path = "/" + result_decoded[loc]
+                    if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename):
+                        result_decoded[loc] = path[1:-len(uri_basename)] + basename
+                    elif not path.endswith("/" + basename):
+                        result_decoded[loc] = os.path.join(path[1:], basename)
         else:
             return None
     result = encodeurl(result_decoded)
@@ -506,7 +518,7 @@ def fetcher_init(d):
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
 
-    _checksum_cache.init_cache(d)
+    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
 
     for m in methods:
         if hasattr(m, "init"):
@@ -534,7 +546,7 @@ def mirror_from_string(data):
         bb.warn('Invalid mirror data %s, should have paired members.' % data)
     return list(zip(*[iter(mirrors)]*2))
 
-def verify_checksum(ud, d, precomputed={}):
+def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
     """
     verify the MD5 and SHA256 checksum for downloaded src
 
@@ -548,20 +560,25 @@ def verify_checksum(ud, d, precomputed={}):
     file against those in the recipe each time, rather than only after
     downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
     """
-
     if ud.ignore_checksums or not ud.method.supports_checksum(ud):
         return {}
 
+    if localpath is None:
+        localpath = ud.localpath
+
     def compute_checksum_info(checksum_id):
         checksum_name = getattr(ud, "%s_name" % checksum_id)
 
         if checksum_id in precomputed:
             checksum_data = precomputed[checksum_id]
         else:
-            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
+            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
 
         checksum_expected = getattr(ud, "%s_expected" % checksum_id)
 
+        if checksum_expected == '':
+            checksum_expected = None
+
         return {
             "id": checksum_id,
             "name": checksum_name,
@@ -581,17 +598,13 @@ def verify_checksum(ud, d, precomputed={}):
             checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
 
     # If no checksum has been provided
-    if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+    if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
         messages = []
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
 
         # If strict checking enabled and neither sum defined, raise error
         if strict == "1":
-            messages.append("No checksum specified for '%s', please add at " \
-                            "least one to the recipe:" % ud.localpath)
-            messages.extend(checksum_lines)
-            logger.error("\n".join(messages))
-            raise NoChecksumError("Missing SRC_URI checksum", ud.url)
+            raise NoChecksumError("\n".join(checksum_lines))
 
         bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
@@ -612,8 +625,8 @@ def verify_checksum(ud, d, precomputed={}):
 
     for ci in checksum_infos:
         if ci["expected"] and ci["expected"] != ci["data"]:
-            messages.append("File: '%s' has %s checksum %s when %s was " \
-                            "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+            messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
+                            "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
             bad_checksum = ci["data"]
 
     if bad_checksum:
@@ -731,13 +744,16 @@ def subprocess_setup():
     # SIGPIPE errors are known issues with gzip/bash
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-def get_autorev(d):
-    #  only not cache src rev in autorev case
+def mark_recipe_nocache(d):
     if d.getVar('BB_SRCREV_POLICY') != "cache":
         d.setVar('BB_DONT_CACHE', '1')
+
+def get_autorev(d):
+    mark_recipe_nocache(d)
+    d.setVar("__BBAUTOREV_SEEN", True)
     return "AUTOINC"
 
-def get_srcrev(d, method_name='sortable_revision'):
+def _get_srcrev(d, method_name='sortable_revision'):
     """
     Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
@@ -751,23 +767,34 @@ def get_srcrev(d, method_name='sortable_revision'):
     that fetcher provides a method with the given name and the same signature as sortable_revision.
     """
 
+    d.setVar("__BBSRCREV_SEEN", "1")
+    recursion = d.getVar("__BBINSRCREV")
+    if recursion:
+        raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
+    d.setVar("__BBINSRCREV", True)
+
     scms = []
+    revs = []
     fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
             scms.append(u)
 
-    if len(scms) == 0:
-        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+    if not scms:
+        d.delVar("__BBINSRCREV")
+        return "", revs
+
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
         autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+        revs.append(rev)
         if len(rev) > 10:
             rev = rev[:10]
+        d.delVar("__BBINSRCREV")
         if autoinc:
-            return "AUTOINC+" + rev
-        return rev
+            return "AUTOINC+" + rev, revs
+        return rev, revs
 
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -783,6 +810,7 @@ def get_srcrev(d, method_name='sortable_revision'):
         ud = urldata[scm]
         for name in ud.names:
             autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
+            revs.append(rev)
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
@@ -799,12 +827,70 @@ def get_srcrev(d, method_name='sortable_revision'):
     if seenautoinc:
         format = "AUTOINC+" + format
 
-    return format
+    d.delVar("__BBINSRCREV")
+    return format, revs
+
+def get_hashvalue(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return " ".join(revs)
+
+def get_pkgv_string(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return pkgv
+
+def get_srcrev(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    if not pkgv:
+        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+    return pkgv
 
 def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
+# Need to export PATH as binary could be in metadata paths
+# rather than host provided
+# Also include some other variables.
+FETCH_EXPORT_VARS = ['HOME', 'PATH',
+                     'HTTP_PROXY', 'http_proxy',
+                     'HTTPS_PROXY', 'https_proxy',
+                     'FTP_PROXY', 'ftp_proxy',
+                     'FTPS_PROXY', 'ftps_proxy',
+                     'NO_PROXY', 'no_proxy',
+                     'ALL_PROXY', 'all_proxy',
+                     'GIT_PROXY_COMMAND',
+                     'GIT_SSH',
+                     'GIT_SSH_COMMAND',
+                     'GIT_SSL_CAINFO',
+                     'GIT_SMART_HTTP',
+                     'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
+                     'SOCKS5_USER', 'SOCKS5_PASSWD',
+                     'DBUS_SESSION_BUS_ADDRESS',
+                     'P4CONFIG',
+                     'SSL_CERT_FILE',
+                     'NODE_EXTRA_CA_CERTS',
+                     'AWS_PROFILE',
+                     'AWS_ACCESS_KEY_ID',
+                     'AWS_SECRET_ACCESS_KEY',
+                     'AWS_ROLE_ARN',
+                     'AWS_WEB_IDENTITY_TOKEN_FILE',
+                     'AWS_DEFAULT_REGION',
+                     'AWS_SESSION_TOKEN',
+                     'GIT_CACHE_PATH',
+                     'REMOTE_CONTAINERS_IPC',
+                     'SSL_CERT_DIR']
+
+def get_fetcher_environment(d):
+    newenv = {}
+    origenv = d.getVar("BB_ORIGENV")
+    for name in bb.fetch2.FETCH_EXPORT_VARS:
+        value = d.getVar(name)
+        if not value and origenv:
+            value = origenv.getVar(name)
+        if value:
+            newenv[name] = value
+    return newenv
+
 def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     """
     Run cmd returning the command output
@@ -813,25 +899,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     Optionally remove the files/directories listed in cleanup upon failure
     """
 
-    # Need to export PATH as binary could be in metadata paths
-    # rather than host provided
-    # Also include some other variables.
-    # FIXME: Should really include all export varaiables?
-    exportvars = ['HOME', 'PATH',
-                  'HTTP_PROXY', 'http_proxy',
-                  'HTTPS_PROXY', 'https_proxy',
-                  'FTP_PROXY', 'ftp_proxy',
-                  'FTPS_PROXY', 'ftps_proxy',
-                  'NO_PROXY', 'no_proxy',
-                  'ALL_PROXY', 'all_proxy',
-                  'GIT_PROXY_COMMAND',
-                  'GIT_SSH',
-                  'GIT_SSL_CAINFO',
-                  'GIT_SMART_HTTP',
-                  'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
-                  'SOCKS5_USER', 'SOCKS5_PASSWD',
-                  'DBUS_SESSION_BUS_ADDRESS',
-                  'P4CONFIG']
+    exportvars = FETCH_EXPORT_VARS
 
     if not cleanup:
         cleanup = []
@@ -868,14 +936,17 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
         success = True
     except bb.process.NotFoundError as e:
-        error_message = "Fetch command %s" % (e.command)
+        error_message = "Fetch command %s not found" % (e.command)
     except bb.process.ExecutionError as e:
         if e.stdout:
             output = "output:\n%s\n%s" % (e.stdout, e.stderr)
         elif e.stderr:
             output = "output:\n%s" % e.stderr
         else:
-            output = "no output"
+            if log:
+                output = "see logfile for output"
+            else:
+                output = "no output"
         error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
     except bb.process.CmdError as e:
         error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
@@ -937,6 +1008,7 @@ def build_mirroruris(origud, mirrors, ld):
 
                 try:
                     newud = FetchData(newuri, ld)
+                    newud.ignore_checksums = True
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
                     logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1046,7 +1118,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
             logger.debug(str(e))
         try:
-            ud.method.clean(ud, ld)
+            if ud.method.cleanup_upon_failure():
+                ud.method.clean(ud, ld)
         except UnboundLocalError:
             pass
         return False
@@ -1057,6 +1130,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
 
 def ensure_symlink(target, link_name):
     if not os.path.exists(link_name):
+        dirname = os.path.dirname(link_name)
+        bb.utils.mkdirhier(dirname)
         if os.path.islink(link_name):
             # Broken symbolic link
             os.unlink(link_name)
@@ -1140,11 +1215,11 @@ def srcrev_internal_helper(ud, d, name):
     pn = d.getVar("PN")
     attempts = []
     if name != '' and pn:
-        attempts.append("SRCREV_%s_pn-%s" % (name, pn))
+        attempts.append("SRCREV_%s:pn-%s" % (name, pn))
     if name != '':
         attempts.append("SRCREV_%s" % name)
     if pn:
-        attempts.append("SRCREV_pn-%s" % pn)
+        attempts.append("SRCREV:pn-%s" % pn)
     attempts.append("SRCREV")
 
     for a in attempts:
@@ -1169,6 +1244,7 @@ def srcrev_internal_helper(ud, d, name):
     if srcrev == "INVALID" or not srcrev:
         raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
     if srcrev == "AUTOINC":
+        d.setVar("__BBAUTOREV_ACTED_UPON", True)
         srcrev = ud.method.latest_revision(ud, d, name)
 
     return srcrev
@@ -1180,23 +1256,21 @@ def get_checksum_file_list(d):
     SRC_URI as a space-separated string
     """
     fetch = Fetch([], d, cache = False, localonly = True)
-
-    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
-
         if ud and isinstance(ud.method, local.Local):
-            paths = ud.method.localpaths(ud, d)
+            found = False
+            paths = ud.method.localfile_searchpaths(ud, d)
             for f in paths:
                 pth = ud.decodedurl
-                if f.startswith(dl_dir):
-                    # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
-                    if os.path.exists(f):
-                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
-                    else:
-                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
+                if os.path.exists(f):
+                    found = True
                 filelist.append(f + ":" + str(os.path.exists(f)))
+            if not found:
+                bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
+                            "\nThe following paths were searched:"
+                            "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
 
     return " ".join(filelist)
 
@@ -1243,18 +1317,13 @@ class FetchData(object):
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
 
             setattr(self, "%s_expected" % checksum_id, checksum_expected)
 
-        for checksum_id in CHECKSUM_LIST:
-            configure_checksum(checksum_id)
-
-        self.ignore_checksums = False
-
         self.names = self.parm.get("name",'default').split(',')
 
         self.method = None
@@ -1276,6 +1345,11 @@ class FetchData(object):
         if hasattr(self.method, "urldata_init"):
             self.method.urldata_init(self, d)
 
+        for checksum_id in CHECKSUM_LIST:
+            configure_checksum(checksum_id)
+
+        self.ignore_checksums = False
+
         if "localpath" in self.parm:
             # if user sets localpath for file, use it instead.
             self.localpath = self.parm["localpath"]
@@ -1355,6 +1429,9 @@ class FetchMethod(object):
         Is localpath something that can be represented by a checksum?
         """
 
+        # We cannot compute checksums for None
+        if urldata.localpath is None:
+            return False
         # We cannot compute checksums for directories
         if os.path.isdir(urldata.localpath):
             return False
@@ -1367,6 +1444,12 @@ class FetchMethod(object):
         """
         return False
 
+    def cleanup_upon_failure(self):
+        """
+        When a fetch fails, should clean() be called?
+        """
+        return True
+
     def verify_donestamp(self, ud, d):
         """
         Verify the donestamp file
@@ -1434,30 +1517,33 @@ class FetchMethod(object):
         cmd = None
 
         if unpack:
+            tar_cmd = 'tar --extract --no-same-owner'
+            if 'striplevel' in urldata.parm:
+                tar_cmd += ' --strip-components=%s' %  urldata.parm['striplevel']
             if file.endswith('.tar'):
-                cmd = 'tar x --no-same-owner -f %s' % file
+                cmd = '%s -f %s' % (tar_cmd, file)
             elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-                cmd = 'tar xz --no-same-owner -f %s' % file
+                cmd = '%s -z -f %s' % (tar_cmd, file)
             elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-                cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
                 cmd = 'gzip -dc %s > %s' % (file, efile)
             elif file.endswith('.bz2'):
                 cmd = 'bzip2 -dc %s > %s' % (file, efile)
             elif file.endswith('.txz') or file.endswith('.tar.xz'):
-                cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.xz'):
                 cmd = 'xz -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.lz'):
-                cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.lz'):
                 cmd = 'lzip -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.7z'):
-                cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
+                cmd = '7z x -so %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.7z'):
                 cmd = '7za x -y %s 1>/dev/null' % file
             elif file.endswith('.tzst') or file.endswith('.tar.zst'):
-                cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file
+                cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.zst'):
                 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
             elif file.endswith('.zip') or file.endswith('.jar'):
@@ -1490,7 +1576,7 @@ class FetchMethod(object):
                         raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url)
                 else:
                     raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
-                cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile)
+                cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
@@ -1506,6 +1592,7 @@ class FetchMethod(object):
             unpackdir = rootdir
 
         if not unpack or not cmd:
+            urldata.unpack_tracer.unpack("file-copy", unpackdir)
             # If file == dest, then avoid any copies, as we already put the file into dest!
             dest = os.path.join(unpackdir, os.path.basename(file))
             if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -1520,6 +1607,8 @@ class FetchMethod(object):
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
                 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
+        else:
+            urldata.unpack_tracer.unpack("archive-extract", unpackdir)
 
         if not cmd:
             return
@@ -1611,12 +1700,61 @@ class FetchMethod(object):
         """
         return []
 
+
+class DummyUnpackTracer(object):
+    """
+    Abstract API definition for a class that traces unpacked source files back
+    to their respective upstream SRC_URI entries, for software composition
+    analysis, license compliance and detailed SBOM generation purposes.
+    User may load their own unpack tracer class (instead of the dummy
+    one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
+    """
+    def start(self, unpackdir, urldata_dict, d):
+        """
+        Start tracing the core Fetch.unpack process, using an index to map
+        unpacked files to each SRC_URI entry.
+        This method is called by Fetch.unpack and it may receive nested calls by
+        gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
+        URLs and by recursively calling Fetch.unpack from new (nested) Fetch
+        instances.
+        """
+        return
+    def start_url(self, url):
+        """Start tracing url unpack process.
+        This method is called by Fetch.unpack before the fetcher-specific unpack
+        method starts, and it may receive nested calls by gitsm and npmsw
+        fetchers.
+        """
+        return
+    def unpack(self, unpack_type, destdir):
+        """
+        Set unpack_type and destdir for current url.
+        This method is called by the fetcher-specific unpack method after url
+        tracing started.
+        """
+        return
+    def finish_url(self, url):
+        """Finish tracing url unpack process and update the file index.
+        This method is called by Fetch.unpack after the fetcher-specific unpack
+        method finished its job, and it may receive nested calls by gitsm
+        and npmsw fetchers.
+        """
+        return
+    def complete(self):
+        """
+        Finish tracing the Fetch.unpack process, and check if all nested
+        Fecth.unpack calls (if any) have been completed; if so, save collected
+        metadata.
+        """
+        return
+
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
-        if len(urls) == 0:
+        if not urls:
             urls = d.getVar("SRC_URI").split()
         self.urls = urls
         self.d = d
@@ -1631,10 +1769,30 @@ class Fetch(object):
         if key in urldata_cache:
             self.ud = urldata_cache[key]
 
+        # the unpack_tracer object needs to be made available to possible nested
+        # Fetch instances (when those are created by gitsm and npmsw fetchers)
+        # so we set it as a global variable
+        global unpack_tracer
+        try:
+            unpack_tracer
+        except NameError:
+            class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
+            if class_path:
+                # use user-defined unpack tracer class
+                import importlib
+                module_name, _, class_name = class_path.rpartition(".")
+                module = importlib.import_module(module_name)
+                class_ = getattr(module, class_name)
+                unpack_tracer = class_()
+            else:
+                # fall back to the dummy/abstract class
+                unpack_tracer = DummyUnpackTracer()
+
         for url in urls:
             if url not in self.ud:
                 try:
                     self.ud[url] = FetchData(url, d, localonly)
+                    self.ud[url].unpack_tracer = unpack_tracer
                 except NonLocalMethod:
                     if localonly:
                         self.ud[url] = None
@@ -1673,6 +1831,7 @@ class Fetch(object):
         network = self.d.getVar("BB_NO_NETWORK")
         premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
 
+        checksum_missing_messages = []
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1684,7 +1843,6 @@ class Fetch(object):
 
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
-
                 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     done = True
                 elif m.try_premirror(ud, self.d):
@@ -1705,7 +1863,9 @@ class Fetch(object):
                     self.d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
-                verified_stamp = m.verify_donestamp(ud, self.d)
+                verified_stamp = False
+                if done:
+                    verified_stamp = m.verify_donestamp(ud, self.d)
                 if not done and (not verified_stamp or m.need_update(ud, self.d)):
                     try:
                         if not trusted_network(self.d, ud.url):
@@ -1735,7 +1895,7 @@ class Fetch(object):
                             logger.debug(str(e))
                         firsterr = e
                         # Remove any incomplete fetch
-                        if not verified_stamp:
+                        if not verified_stamp and m.cleanup_upon_failure():
                             m.clean(ud, self.d)
                         logger.debug("Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
@@ -1754,17 +1914,28 @@ class Fetch(object):
                     raise ChecksumError("Stale Error Detected")
 
             except BBFetchException as e:
-                if isinstance(e, ChecksumError):
+                if isinstance(e, NoChecksumError):
+                    (message, _) = e.args
+                    checksum_missing_messages.append(message)
+                    continue
+                elif isinstance(e, ChecksumError):
                     logger.error("Checksum failure fetching %s" % u)
                 raise
 
             finally:
                 if ud.lockfile:
                     bb.utils.unlockfile(lf)
+        if checksum_missing_messages:
+            logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
+            raise BBFetchException("There was some missing checksums in the recipe")
 
     def checkstatus(self, urls=None):
         """
-        Check all urls exist upstream
+        Check all URLs exist upstream.
+
+        Returns None if the URLs exist, raises FetchError if the check wasn't
+        successful but there wasn't an error (such as file not found), and
+        raises other exceptions in error cases.
         """
 
         if not urls:
@@ -1787,7 +1958,7 @@ class Fetch(object):
                     ret = m.try_mirrors(self, ud, self.d, mirrors, True)
 
             if not ret:
-                raise FetchError("URL %s doesn't work" % u, u)
+                raise FetchError("URL doesn't work", u)
 
     def unpack(self, root, urls=None):
         """
@@ -1797,6 +1968,8 @@ class Fetch(object):
         if not urls:
             urls = self.urls
 
+        unpack_tracer.start(root, self.ud, self.d)
+
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1804,11 +1977,15 @@ class Fetch(object):
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
 
+            unpack_tracer.start_url(u)
             ud.method.unpack(ud, root, self.d)
+            unpack_tracer.finish_url(u)
 
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+        unpack_tracer.complete()
+
     def clean(self, urls=None):
         """
         Clean files that the fetcher gets or places
@@ -1908,6 +2085,9 @@ from . import repo
 from . import clearcase
 from . import npm
 from . import npmsw
+from . import az
+from . import crate
+from . import gcp
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1927,3 +2107,6 @@ methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
+methods.append(az.Az())
+methods.append(crate.Crate())
+methods.append(gcp.GCP())
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py
new file mode 100644
index 0000000000..3ccc594c22
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/az.py
@@ -0,0 +1,93 @@
+"""
+BitBake 'Fetch' Azure Storage implementation
+
+"""
+
+# Copyright (C) 2021 Alejandro Hernandez Samaniego
+#
+# Based on bb.fetch2.wget:
+# Copyright (C) 2003, 2004  Chris Larson
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import shlex
+import os
+import bb
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import logger
+from   bb.fetch2.wget import Wget
+
+
+class Az(Wget):
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched from Azure Storage
+        """
+        return ud.type in ['az']
+
+
+    def checkstatus(self, fetch, ud, d, try_again=True):
+
+        # checkstatus discards parameters either way, we need to do this before adding the SAS
+        ud.url = ud.url.replace('az://','https://').split(';')[0]
+
+        az_sas = d.getVar('AZ_SAS')
+        if az_sas and az_sas not in ud.url:
+            ud.url += az_sas
+
+        return Wget.checkstatus(self, fetch, ud, d, try_again)
+
+    # Override download method, include retries
+    def download(self, ud, d, retries=3):
+        """Fetch urls"""
+
+        # If were reaching the account transaction limit we might be refused a connection,
+        # retrying allows us to avoid false negatives since the limit changes over time
+        fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+
+        # We need to provide a localpath to avoid wget using the SAS
+        # ud.localfile either has the downloadfilename or ud.path
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
+
+
+        if ud.user and ud.pswd:
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+
+        # Check if a Shared Access Signature was given and use it
+        az_sas = d.getVar('AZ_SAS')
+
+        if az_sas:
+            azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
+        else:
+            azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+
+        if os.path.exists(ud.localpath):
+            # file exists, but we didnt complete it.. trying again.
+            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+        else:
+            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+
+        try:
+            self._runwget(ud, d, fetchcmd, False)
+        except FetchError as e:
+            # Azure fails on handshake sometimes when using wget after some stress, producing a
+            # FetchError from the fetcher, if the artifact exists retyring should succeed
+            if 'Unable to establish SSL connection' in str(e):
+                logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
+                self.download(ud, d, retries -1)
+
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
+
+        return True
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
new file mode 100644
index 0000000000..01d49435c3
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -0,0 +1,141 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementation for crates.io
+"""
+
+# Copyright (C) 2016 Doug Goldstein
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import hashlib
+import json
+import os
+import subprocess
+import bb
+from   bb.fetch2 import logger, subprocess_setup, UnpackError
+from   bb.fetch2.wget import Wget
+
+
+class Crate(Wget):
+
+    """Class to fetch crates via wget"""
+
+    def _cargo_bitbake_path(self, rootdir):
+        return os.path.join(rootdir, "cargo_home", "bitbake")
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url is for this fetcher
+        """
+        return ud.type in ['crate']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        """
+        Sets up to download the respective crate from crates.io
+        """
+
+        if ud.type == 'crate':
+            self._crate_urldata_init(ud, d)
+
+        super(Crate, self).urldata_init(ud, d)
+
+    def _crate_urldata_init(self, ud, d):
+        """
+        Sets up the download for a crate
+        """
+
+        # URL syntax is: crate://NAME/VERSION
+        # break the URL apart by /
+        parts = ud.url.split('/')
+        if len(parts) < 5:
+            raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
+
+        # version is expected to be the last token
+        # but ignore possible url parameters which will be used
+        # by the top fetcher class
+        version = parts[-1].split(";")[0]
+        # second to last field is name
+        name = parts[-2]
+        # host (this is to allow custom crate registries to be specified
+        host = '/'.join(parts[2:-2])
+
+        # if using upstream just fix it up nicely
+        if host == 'crates.io':
+            host = 'crates.io/api/v1/crates'
+
+        ud.url = "https://%s/%s/%s/download" % (host, name, version)
+        ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
+        if 'name' not in ud.parm:
+            ud.parm['name'] = '%s-%s' % (name, version)
+
+        logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
+
+    def unpack(self, ud, rootdir, d):
+        """
+        Uses the crate to build the necessary paths for cargo to utilize it
+        """
+        if ud.type == 'crate':
+            return self._crate_unpack(ud, rootdir, d)
+        else:
+            super(Crate, self).unpack(ud, rootdir, d)
+
+    def _crate_unpack(self, ud, rootdir, d):
+        """
+        Unpacks a crate
+        """
+        thefile = ud.localpath
+
+        # possible metadata we need to write out
+        metadata = {}
+
+        # change to the rootdir to unpack but save the old working dir
+        save_cwd = os.getcwd()
+        os.chdir(rootdir)
+
+        bp = d.getVar('BP')
+        if bp == ud.parm.get('name'):
+            cmd = "tar -xz --no-same-owner -f %s" % thefile
+            ud.unpack_tracer.unpack("crate-extract", rootdir)
+        else:
+            cargo_bitbake = self._cargo_bitbake_path(rootdir)
+            ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
+
+            cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
+
+            # ensure we've got these paths made
+            bb.utils.mkdirhier(cargo_bitbake)
+
+            # generate metadata necessary
+            with open(thefile, 'rb') as f:
+                # get the SHA256 of the original tarball
+                tarhash = hashlib.sha256(f.read()).hexdigest()
+
+            metadata['files'] = {}
+            metadata['package'] = tarhash
+
+        path = d.getVar('PATH')
+        if path:
+            cmd = "PATH=\"%s\" %s" % (path, cmd)
+        bb.note("Unpacking %s to %s/" % (thefile, os.getcwd()))
+
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+        os.chdir(save_cwd)
+
+        if ret != 0:
+            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
+
+        # if we have metadata to write out..
+        if len(metadata) > 0:
+            cratepath = os.path.splitext(os.path.basename(thefile))[0]
+            bbpath = self._cargo_bitbake_path(rootdir)
+            mdfile = '.cargo-checksum.json'
+            mdpath = os.path.join(bbpath, cratepath, mdfile)
+            with open(mdpath, "w") as f:
+                json.dump(metadata, f)
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
new file mode 100644
index 0000000000..eb3e0c6a6b
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -0,0 +1,102 @@
+"""
+BitBake 'Fetch' implementation for Google Cloup Platform Storage.
+
+Class for fetching files from Google Cloud Storage using the
+Google Cloud Storage Python Client. The GCS Python Client must
+be correctly installed, configured and authenticated prior to use.
+Additionally, gsutil must also be installed.
+
+"""
+
+# Copyright (C) 2023, Snap Inc.
+#
+# Based in part on bb.fetch2.s3:
+#    Copyright (C) 2017 Andre McCurdy
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
+
+class GCP(FetchMethod):
+    """
+    Class to fetch urls via GCP's Python API.
+    """
+    def __init__(self):
+        self.gcp_client = None
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with GCP.
+        """
+        return ud.type in ['gs']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.basecmd = "gsutil stat"
+
+    def get_gcp_client(self):
+        from google.cloud import storage
+        self.gcp_client = storage.Client(project=None)
+
+    def download(self, ud, d):
+        """
+        Fetch urls using the GCP API.
+        Assumes localpath was called first.
+        """
+        logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        blob = self.gcp_client.bucket(ud.host).blob(path)
+        blob.download_to_filename(ud.localpath)
+
+        # Additional sanity checks copied from the wget class (although there
+        # are no known issues which mean these are required, treat the GCP API
+        # tool with a little healthy suspicion).
+        if not os.path.exists(ud.localpath):
+            raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
+
+        return True
+
+    def checkstatus(self, fetch, ud, d):
+        """
+        Check the status of a URL.
+        """
+        logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
+            raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
+        else:
+            return True
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e3ba80a3f5..c7ff769fdf 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -44,13 +44,27 @@ Supported SRC_URI options are:
 
 - nobranch
    Don't check the SHA validation for branch. set this option for the recipe
-   referring to commit which is valid in tag instead of branch.
+   referring to commit which is valid in any namespace (branch, tag, ...)
+   instead of branch.
    The default is "0", set nobranch=1 if needed.
 
+- subpath
+   Limit the checkout to a specific subpath of the tree.
+   By default, checkout the whole tree, set subpath=<path> if needed
+
+- destsuffix
+   The name of the path in which to place the checkout.
+   By default, the path is git/, set destsuffix=<suffix> if needed
+
 - usehead
    For local git:// urls to use the current branch HEAD as the revision for use with
    AUTOREV. Implies nobranch.
 
+- lfs
+    Enable the checkout to use LFS for large files. This will download all LFS files
+    in the download step, as the unpack step does not have network access.
+    The default is "1", set lfs=0 to skip.
+
 """
 
 # Copyright (C) 2005 Richard Purdie
@@ -64,14 +78,20 @@ import fnmatch
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import tempfile
 import bb
 import bb.progress
+from contextlib import contextmanager
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
+from   bb.fetch2 import trusted_network
+
 
+sha1_re = re.compile(r'^[0-9a-f]{40}$')
+slash_re = re.compile(r"/+")
 
 class GitProgressHandler(bb.progress.LineFilterProgressHandler):
     """Extract progress information from git output"""
@@ -130,6 +150,9 @@ class Git(FetchMethod):
     def supports_checksum(self, urldata):
         return False
 
+    def cleanup_upon_failure(self):
+        return False
+
     def urldata_init(self, ud, d):
         """
         init git specific variable within url data
@@ -141,6 +164,11 @@ class Git(FetchMethod):
             ud.proto = 'file'
         else:
             ud.proto = "git"
+        if ud.host == "github.com" and ud.proto == "git":
+            # github stopped supporting git protocol
+            # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
+            ud.proto = "https"
+            bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
 
         if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
             raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -164,11 +192,18 @@ class Git(FetchMethod):
             ud.nocheckout = 1
   
         ud.unresolvedrev = {}
-        branches = ud.parm.get("branch", "master").split(',')
+        branches = ud.parm.get("branch", "").split(',')
+        if branches == [""] and not ud.nobranch:
+            bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
+            branches = ["master"]
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
 
-        ud.cloneflags = "-s -n"
+        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
+
+        ud.cloneflags = "-n"
+        if not ud.noshared:
+            ud.cloneflags += " -s"
         if ud.bareclone:
             ud.cloneflags += " --mirror"
 
@@ -227,7 +262,7 @@ class Git(FetchMethod):
             for name in ud.names:
                 ud.unresolvedrev[name] = 'HEAD'
 
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all"
 
         write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
         ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -236,20 +271,20 @@ class Git(FetchMethod):
         ud.setup_revisions(d)
 
         for name in ud.names:
-            # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
-            if not ud.revisions[name] or len(ud.revisions[name]) != 40  or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
+            # Ensure any revision that doesn't look like a SHA-1 is translated into one
+            if not sha1_re.match(ud.revisions[name] or ''):
                 if ud.revisions[name]:
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
-        # for rebaseable git repo, it is necessary to keep mirror tar ball
-        # per revision, so that even the revision disappears from the
+        # For a rebaseable git repo, it is necessary to keep a mirror tar ball
+        # per revision, so that even if the revision disappears from the
         # upstream repo in the future, the mirror will remain intact and still
-        # contains the revision
+        # contain the revision
         if ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
@@ -293,7 +328,10 @@ class Git(FetchMethod):
         return ud.clonedir
 
     def need_update(self, ud, d):
-        return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+        return self.clonedir_need_update(ud, d) \
+                or self.shallow_tarball_need_update(ud) \
+                or self.tarball_need_update(ud) \
+                or self.lfs_need_update(ud, d)
 
     def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
@@ -305,6 +343,15 @@ class Git(FetchMethod):
                 return True
         return False
 
+    def lfs_need_update(self, ud, d):
+        if self.clonedir_need_update(ud, d):
+            return True
+
+        for name in ud.names:
+            if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
+                return True
+        return False
+
     def clonedir_need_shallow_revs(self, ud, d):
         for rev in ud.shallow_revs:
             try:
@@ -324,6 +371,16 @@ class Git(FetchMethod):
         # is not possible
         if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
             return True
+        # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
+        # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
+        # we need to try premirrors first as using upstream is destined to fail.
+        if not trusted_network(d, ud.url):
+            return True
+        # the following check is to ensure incremental fetch in downloads, this is
+        # because the premirror might be old and does not contain the new rev required,
+        # and this will cause a total removal and new clone. So if we can reach to
+        # network, we prefer upstream over premirror, though the premirror might contain
+        # the new rev.
         if os.path.exists(ud.clonedir):
             return False
         return True
@@ -337,17 +394,54 @@ class Git(FetchMethod):
         if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
             ud.localpath = ud.fullshallow
             return
-        elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
-            bb.utils.mkdirhier(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
-
+        elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
+            if not os.path.exists(ud.clonedir):
+                bb.utils.mkdirhier(ud.clonedir)
+                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
+            else:
+                tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
+                output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+                if 'mirror' in output:
+                    runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
+                runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
+                fetch_cmd = "LANG=C %s fetch -f --update-head-ok  --progress mirror " % (ud.basecmd)
+                runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
         repourl = self._get_repo_url(ud)
 
+        needs_clone = False
+        if os.path.exists(ud.clonedir):
+            # The directory may exist, but not be the top level of a bare git
+            # repository in which case it needs to be deleted and re-cloned.
+            try:
+                # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
+                output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
+                toplevel = output.rstrip()
+
+                if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
+                    logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
+                    needs_clone = True
+            except bb.fetch2.FetchError as e:
+                logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
+                needs_clone = True
+            except FileNotFoundError as e:
+                logger.warning("%s", e)
+                needs_clone = True
+
+            if needs_clone:
+                shutil.rmtree(ud.clonedir)
+        else:
+            needs_clone = True
+
         # If the repo still doesn't exist, fallback to cloning it
-        if not os.path.exists(ud.clonedir):
-            # We do this since git will use a "-l" option automatically for local urls where possible
+        if needs_clone:
+            # We do this since git will use a "-l" option automatically for local urls where possible,
+            # but it doesn't work when git/objects is a symlink, only works when it is a directory.
             if repourl.startswith("file://"):
-                repourl = repourl[7:]
+                repourl_path = repourl[7:]
+                objects = os.path.join(repourl_path, 'objects')
+                if os.path.isdir(objects) and not os.path.islink(objects):
+                    repourl = repourl_path
             clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
@@ -361,7 +455,11 @@ class Git(FetchMethod):
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
-            fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+
+            if ud.nobranch:
+                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+            else:
+                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -384,17 +482,16 @@ class Git(FetchMethod):
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
-        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+        if self.lfs_need_update(ud, d):
             # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
-            # of all LFS blobs needed at the the srcrev.
+            # of all LFS blobs needed at the srcrev.
             #
             # It would be nice to just do this inline here by running 'git-lfs fetch'
             # on the bare clonedir, but that operation requires a working copy on some
             # releases of Git LFS.
-            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
-            try:
+            with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
                 # Do the checkout. This implicitly involves a Git LFS fetch.
-                self.unpack(ud, tmpdir, d)
+                Git.unpack(self, ud, tmpdir, d)
 
                 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
                 # the bare clonedir.
@@ -408,12 +505,24 @@ class Git(FetchMethod):
                 # Only do this if the unpack resulted in a .git/lfs directory being
                 # created; this only happens if at least one blob needed to be
                 # downloaded.
-                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
-                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
-            finally:
-                bb.utils.remove(tmpdir, recurse=True)
+                if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
+                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
 
     def build_mirror_data(self, ud, d):
+
+        # Create as a temp file and move atomically into position to avoid races
+        @contextmanager
+        def create_atomic(filename):
+            fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+            try:
+                yield tfile
+                umask = os.umask(0o666)
+                os.umask(umask)
+                os.chmod(tfile, (0o666 & ~umask))
+                os.rename(tfile, filename)
+            finally:
+                os.close(fd)
+
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
                 if os.path.islink(ud.fullshallow):
@@ -424,7 +533,8 @@ class Git(FetchMethod):
                     self.clone_shallow_local(ud, shallowclone, d)
 
                     logger.info("Creating tarball of git repository")
-                    runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
+                    with create_atomic(ud.fullshallow) as tfile:
+                        runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
                     runfetchcmd("touch %s.done" % ud.fullshallow, d)
                 finally:
                     bb.utils.remove(tempdir, recurse=True)
@@ -433,7 +543,11 @@ class Git(FetchMethod):
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
+            with create_atomic(ud.fullmirror) as tfile:
+                mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
+                        quiet=True, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
+                        % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
@@ -495,18 +609,31 @@ class Git(FetchMethod):
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            readpathspec = ":%s" % subdir
-            def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
-        else:
-            readpathspec = ""
-            def_destsuffix = "git/"
+        subdir = ud.parm.get("subdir")
+        subpath = ud.parm.get("subpath")
+        readpathspec = ""
+        def_destsuffix = "git/"
+
+        if subpath:
+            readpathspec = ":%s" % subpath
+            def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
+
+        if subdir:
+            # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
+            if os.path.isabs(subdir):
+                if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
+                    raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
+                destdir = subdir
+            else:
+                destdir = os.path.join(destdir, subdir)
+            def_destsuffix = ""
 
         destsuffix = ud.parm.get("destsuffix", def_destsuffix)
         destdir = ud.destdir = os.path.join(destdir, destsuffix)
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
+        if not ud.bareclone:
+            ud.unpack_tracer.unpack("git", destdir)
 
         need_lfs = self._need_lfs(ud)
 
@@ -516,13 +643,12 @@ class Git(FetchMethod):
         source_found = False
         source_error = []
 
-        if not source_found:
-            clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
-            if clonedir_is_up_to_date:
-                runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
-                source_found = True
-            else:
-                source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+        clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+        if clonedir_is_up_to_date:
+            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+            source_found = True
+        else:
+            source_error.append("clone directory not available or not up to date: " + ud.clonedir)
 
         if not source_found:
             if ud.shallow:
@@ -546,9 +672,11 @@ class Git(FetchMethod):
                 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
             elif not need_lfs:
                 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
+            else:
+                runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
 
         if not ud.nocheckout:
-            if subdir != "":
+            if subpath:
                 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
@@ -597,6 +725,35 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _lfs_objects_downloaded(self, ud, d, name, wd):
+        """
+        Verifies whether the LFS objects for requested revisions have already been downloaded
+        """
+        # Bail out early if this repository doesn't use LFS
+        if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd):
+            return True
+
+        # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
+        # existence.
+        # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
+        cmd = "%s lfs ls-files -l %s" \
+                % (ud.basecmd, ud.revisions[name])
+        output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
+        # Do not do any further matching if no objects are managed by LFS
+        if not output:
+            return True
+
+        # Match all lines beginning with the hexadecimal OID
+        oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
+        for line in output.split("\n"):
+            oid = re.search(oid_regex, line)
+            if not oid:
+                bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
+            if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
+                return False
+
+        return True
+
     def _need_lfs(self, ud):
         return ud.parm.get("lfs", "1") == "1"
 
@@ -605,13 +762,11 @@ class Git(FetchMethod):
         Check if the repository has 'lfs' (large file) content
         """
 
-        if not ud.nobranch:
-            branchname = ud.branches[ud.names[0]]
-        else:
-            branchname = "master"
-
-        # The bare clonedir doesn't use the remote names; it has the branch immediately.
-        if wd == ud.clonedir:
+        if ud.nobranch:
+            # If no branch is specified, use the current git commit
+            refname = self._build_revision(ud, d, ud.names[0])
+        elif wd == ud.clonedir:
+            # The bare clonedir doesn't use the remote names; it has the branch immediately.
             refname = ud.branches[ud.names[0]]
         else:
             refname = "origin/%s" % ud.branches[ud.names[0]]
@@ -654,7 +809,6 @@ class Git(FetchMethod):
         Return a unique key for the url
         """
         # Collapse adjacent slashes
-        slash_re = re.compile(r"/+")
         return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
 
     def _lsremote(self, ud, d, search):
@@ -687,6 +841,12 @@ class Git(FetchMethod):
         """
         Compute the HEAD revision for the url
         """
+        if not d.getVar("__BBSRCREV_SEEN"):
+            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
+
+        # Ensure we mark as not cached
+        bb.fetch2.mark_recipe_nocache(d)
+
         output = self._lsremote(ud, d, "")
         # Tags of the form ^{} may not work, need to fallback to other form
         if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
@@ -711,38 +871,42 @@ class Git(FetchMethod):
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
             bb.note("Could not list remote: %s" % str(e))
             return pupver
 
+        rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
+        pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
+        nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
+
         verstring = ""
-        revision = ""
         for line in output.split("\n"):
             if not line:
                 break
 
-            tag_head = line.split("/")[-1]
+            m = rev_tag_re.match(line)
+            if not m:
+                continue
+
+            (revision, tag) = m.groups()
+
             # Ignore non-released branches
-            m = re.search(r"(alpha|beta|rc|final)+", tag_head)
-            if m:
+            if nonrel_re.search(tag):
                 continue
 
             # search for version in the line
-            tag = tagregex.search(tag_head)
-            if tag is None:
+            m = pver_re.search(tag)
+            if not m:
                 continue
 
-            tag = tag.group('pver')
-            tag = tag.replace("_", ".")
+            pver = m.group('pver').replace("_", ".")
 
-            if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+            if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
                 continue
 
-            verstring = tag
-            revision = line.split()[0]
+            verstring = pver
             pupver = (verstring, revision)
 
         return pupver
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index a4527bf364..f7f3af7212 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -88,9 +88,9 @@ class GitSM(Git):
                 subrevision[m] = module_hash.split()[2]
 
                 # Convert relative to absolute uri based on parent uri
-                if uris[m].startswith('..'):
+                if  uris[m].startswith('..') or uris[m].startswith('./'):
                     newud = copy.copy(ud)
-                    newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
+                    newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
                     uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
@@ -115,10 +115,21 @@ class GitSM(Git):
                     # This has to be a file reference
                     proto = "file"
                     url = "gitsm://" + uris[module]
+            if url.endswith("{}{}".format(ud.host, ud.path)):
+                raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
+                                           "Consider using git fetcher instead.")
 
             url += ';protocol=%s' % proto
             url += ";name=%s" % module
             url += ";subpath=%s" % module
+            url += ";nobranch=1"
+            url += ";lfs=%s" % self._need_lfs(ud)
+            # Note that adding "user=" here to give credentials to the
+            # submodule is not supported. Since using SRC_URI to give git://
+            # URL a password is not supported, one have to use one of the
+            # recommended way (eg. ~/.netrc or SSH config) which does specify
+            # the user (See comment in git.py).
+            # So, we will not take patches adding "user=" support here.
 
             ld = d.createCopy()
             # Not necessary to set SRC_URI, since we're passing the URI to
@@ -140,16 +151,6 @@ class GitSM(Git):
         if Git.need_update(self, ud, d):
             return True
 
-        try:
-            # Check for the nugget dropped by the download operation
-            known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
-                                        (ud.basecmd), d, workdir=ud.clonedir)
-
-            if ud.revisions[ud.names[0]] in known_srcrevs.split():
-                return False
-        except bb.fetch2.FetchError:
-            pass
-
         need_update_list = []
         def need_update_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
@@ -172,13 +173,8 @@ class GitSM(Git):
             shutil.rmtree(tmpdir)
         else:
             self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
-            if len(need_update_list) == 0:
-                # We already have the required commits of all submodules. Drop
-                # a nugget so we don't need to check again.
-                runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
 
-        if len(need_update_list) > 0:
+        if need_update_list:
             logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
@@ -209,9 +205,6 @@ class GitSM(Git):
             shutil.rmtree(tmpdir)
         else:
             self.process_submodules(ud, ud.clonedir, download_submodule, d)
-            # Drop a nugget for the srcrev we've fetched (used by need_update)
-            runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                        (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
 
     def unpack(self, ud, destdir, d):
         def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -225,6 +218,10 @@ class GitSM(Git):
 
             try:
                 newfetch = Fetch([url], d, cache=False)
+                # modpath is needed by unpack tracer to calculate submodule
+                # checkout dir
+                new_ud = newfetch.ud[url]
+                new_ud.modpath = modpath
                 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
             except Exception as e:
                 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
@@ -250,10 +247,12 @@ class GitSM(Git):
         ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
 
         if not ud.bareclone and ret:
-            # All submodules should already be downloaded and configured in the tree.  This simply sets
-            # up the configuration and checks out the files.  The main project config should remain
-            # unmodified, and no download from the internet should occur.
-            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+            # All submodules should already be downloaded and configured in the tree.  This simply
+            # sets up the configuration and checks out the files.  The main project config should
+            # remain unmodified, and no download from the internet should occur. As such, lfs smudge
+            # should also be skipped as these files were already smudged in the fetch stage if lfs
+            # was enabled.
+            runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
 
     def implicit_urldata(self, ud, d):
         import shutil, subprocess, tempfile
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 063e13008a..cbff8c490c 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -242,6 +242,7 @@ class Hg(FetchMethod):
         revflag = "-r %s" % ud.revision
         subdir = ud.parm.get("destsuffix", ud.module)
         codir = "%s/%s" % (destdir, subdir)
+        ud.unpack_tracer.unpack("hg", codir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index e7d1c8c58f..7d7668110e 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -41,9 +41,9 @@ class Local(FetchMethod):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
-        return self.localpaths(urldata, d)[-1]
+        return self.localfile_searchpaths(urldata, d)[-1]
 
-    def localpaths(self, urldata, d):
+    def localfile_searchpaths(self, urldata, d):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
@@ -51,18 +51,14 @@ class Local(FetchMethod):
         path = urldata.decodedurl
         newpath = path
         if path[0] == "/":
+            logger.debug2("Using absolute %s" % (path))
             return [path]
         filespath = d.getVar('FILESPATH')
         if filespath:
             logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
+            logger.debug2("Using %s for %s" % (newpath, path))
             searched.extend(hist)
-        if not os.path.exists(newpath):
-            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
-            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
-            bb.utils.mkdirhier(os.path.dirname(dldirfile))
-            searched.append(dldirfile)
-            return searched
         return searched
 
     def need_update(self, ud, d):
@@ -78,9 +74,7 @@ class Local(FetchMethod):
             filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            locations.append(d.getVar("DL_DIR"))
-
-            msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
+            msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
 
         return True
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 47898509ff..15f3f19bc8 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -44,17 +44,24 @@ def npm_package(package):
     """Convert the npm package name to remove unsupported character"""
     # Scoped package names (with the @) use the same naming convention
     # as the 'npm pack' command.
-    if package.startswith("@"):
-        return re.sub("/", "-", package[1:])
-    return package
+    name = re.sub("/", "-", package)
+    name = name.lower()
+    name = re.sub(r"[^\-a-z0-9]", "", name)
+    name = name.strip("-")
+    return name
+
 
 def npm_filename(package, version):
     """Get the filename of a npm package"""
     return npm_package(package) + "-" + version + ".tgz"
 
-def npm_localfile(package, version):
+def npm_localfile(package, version=None):
     """Get the local filename of a npm package"""
-    return os.path.join("npm2", npm_filename(package, version))
+    if version is not None:
+        filename = npm_filename(package, version)
+    else:
+        filename = package
+    return os.path.join("npm2", filename)
 
 def npm_integrity(integrity):
     """
@@ -69,41 +76,52 @@ def npm_unpack(tarball, destdir, d):
     bb.utils.mkdirhier(destdir)
     cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
     cmd += " --no-same-owner"
+    cmd += " --delay-directory-restore"
     cmd += " --strip-components=1"
     runfetchcmd(cmd, d, workdir=destdir)
+    runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
 
 class NpmEnvironment(object):
     """
     Using a npm config file seems more reliable than using cli arguments.
     This class allows to create a controlled environment for npm commands.
     """
-    def __init__(self, d, configs=None):
+    def __init__(self, d, configs=[], npmrc=None):
         self.d = d
-        self.configs = configs
+
+        self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
+        for key, value in configs:
+            self.user_config.write("%s=%s\n" % (key, value))
+
+        if npmrc:
+            self.global_config_name = npmrc
+        else:
+            self.global_config_name = "/dev/null"
+
+    def __del__(self):
+        if self.user_config:
+            self.user_config.close()
 
     def run(self, cmd, args=None, configs=None, workdir=None):
         """Run npm command in a controlled environment"""
         with tempfile.TemporaryDirectory() as tmpdir:
             d = bb.data.createCopy(self.d)
+            d.setVar("PATH", d.getVar("PATH"))  # PATH might contain $HOME - evaluate it before patching
             d.setVar("HOME", tmpdir)
 
-            cfgfile = os.path.join(tmpdir, "npmrc")
-
             if not workdir:
                 workdir = tmpdir
 
             def _run(cmd):
-                cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd
-                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd
+                cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
+                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
                 return runfetchcmd(cmd, d, workdir=workdir)
 
-            if self.configs:
-                for key, value in self.configs:
-                    _run("npm config set %s %s" % (key, shlex.quote(value)))
-
             if configs:
+                bb.warn("Use of configs argument of NpmEnvironment.run() function"
+                    " is deprecated. Please use args argument instead.")
                 for key, value in configs:
-                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+                    cmd += " --%s=%s" % (key, shlex.quote(value))
 
             if args:
                 for key, value in args:
@@ -142,12 +160,12 @@ class Npm(FetchMethod):
             raise ParameterError("Invalid 'version' parameter", ud.url)
 
         # Extract the 'registry' part of the url
-        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
+        ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
 
         # Using the 'downloadfilename' parameter as local filename
         # or the npm package name.
         if "downloadfilename" in ud.parm:
-            ud.localfile = d.expand(ud.parm["downloadfilename"])
+            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
         else:
             ud.localfile = npm_localfile(ud.package, ud.version)
 
@@ -165,14 +183,14 @@ class Npm(FetchMethod):
 
     def _resolve_proxy_url(self, ud, d):
         def _npm_view():
-            configs = []
-            configs.append(("json", "true"))
-            configs.append(("registry", ud.registry))
+            args = []
+            args.append(("json", "true"))
+            args.append(("registry", ud.registry))
             pkgver = shlex.quote(ud.package + "@" + ud.version)
             cmd = ud.basecmd + " view %s" % pkgver
             env = NpmEnvironment(d)
             check_network_access(d, cmd, ud.registry)
-            view_string = env.run(cmd, configs=configs)
+            view_string = env.run(cmd, args=args)
 
             if not view_string:
                 raise FetchError("Unavailable package %s" % pkgver, ud.url)
@@ -280,6 +298,7 @@ class Npm(FetchMethod):
         destsuffix = ud.parm.get("destsuffix", "npm")
         destdir = os.path.join(rootdir, destsuffix)
         npm_unpack(ud.localpath, destdir, d)
+        ud.unpack_tracer.unpack("npm", destdir)
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index 0c3511d8ab..ff5f8dc755 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -24,11 +24,14 @@ import bb
 from bb.fetch2 import Fetch
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import ParameterError
+from bb.fetch2 import runfetchcmd
 from bb.fetch2 import URI
 from bb.fetch2.npm import npm_integrity
 from bb.fetch2.npm import npm_localfile
 from bb.fetch2.npm import npm_unpack
 from bb.utils import is_semver
+from bb.utils import lockfile
+from bb.utils import unlockfile
 
 def foreach_dependencies(shrinkwrap, callback=None, dev=False):
     """
@@ -38,8 +41,9 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
         with:
             name = the package name (string)
             params = the package parameters (dictionary)
-            deptree = the package dependency tree (array of strings)
+            destdir = the destination of the package (string)
     """
+    # For handling old style dependencies entries in shinkwrap files
     def _walk_deps(deps, deptree):
         for name in deps:
             subtree = [*deptree, name]
@@ -49,9 +53,22 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
                     continue
                 elif deps[name].get("bundled", False):
                     continue
-                callback(name, deps[name], subtree)
-
-    _walk_deps(shrinkwrap.get("dependencies", {}), [])
+                destsubdirs = [os.path.join("node_modules", dep) for dep in subtree]
+                destsuffix = os.path.join(*destsubdirs)
+                callback(name, deps[name], destsuffix)
+
+    # packages entry means new style shrinkwrap file, else use dependencies
+    packages = shrinkwrap.get("packages", None)
+    if packages is not None:
+        for package in packages:
+            if package != "":
+                name = package.split('node_modules/')[-1]
+                package_infos = packages.get(package, {})
+                if dev == False and package_infos.get("dev", False):
+                    continue
+                callback(name, package_infos, package)
+    else:
+        _walk_deps(shrinkwrap.get("dependencies", {}), [])
 
 class NpmShrinkWrap(FetchMethod):
     """Class to fetch all package from a shrinkwrap file"""
@@ -72,19 +89,22 @@ class NpmShrinkWrap(FetchMethod):
         # Resolve the dependencies
         ud.deps = []
 
-        def _resolve_dependency(name, params, deptree):
+        def _resolve_dependency(name, params, destsuffix):
             url = None
             localpath = None
             extrapaths = []
-            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
-            destsuffix = os.path.join(*destsubdirs)
+            unpack = True
 
             integrity = params.get("integrity", None)
             resolved = params.get("resolved", None)
             version = params.get("version", None)
 
             # Handle registry sources
-            if is_semver(version) and resolved and integrity:
+            if is_semver(version) and integrity:
+                # Handle duplicate dependencies without url
+                if not resolved:
+                    return
+
                 localfile = npm_localfile(name, version)
 
                 uri = URI(resolved)
@@ -109,7 +129,7 @@ class NpmShrinkWrap(FetchMethod):
 
             # Handle http tarball sources
             elif version.startswith("http") and integrity:
-                localfile = os.path.join("npm2", os.path.basename(version))
+                localfile = npm_localfile(os.path.basename(version))
 
                 uri = URI(version)
                 uri.params["downloadfilename"] = localfile
@@ -121,8 +141,28 @@ class NpmShrinkWrap(FetchMethod):
 
                 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
 
+            # Handle local tarball and link sources
+            elif version.startswith("file"):
+                localpath = version[5:]
+                if not version.endswith(".tgz"):
+                    unpack = False
+
             # Handle git sources
-            elif version.startswith("git"):
+            elif version.startswith(("git", "bitbucket","gist")) or (
+                not version.endswith((".tgz", ".tar", ".tar.gz"))
+                and not version.startswith((".", "@", "/"))
+                and "/" in version
+            ):
+                if version.startswith("github:"):
+                    version = "git+https://github.com/" + version[len("github:"):]
+                elif version.startswith("gist:"):
+                    version = "git+https://gist.github.com/" + version[len("gist:"):]
+                elif version.startswith("bitbucket:"):
+                    version = "git+https://bitbucket.org/" + version[len("bitbucket:"):]
+                elif version.startswith("gitlab:"):
+                    version = "git+https://gitlab.com/" + version[len("gitlab:"):]
+                elif not version.startswith(("git+","git:")):
+                    version = "git+https://github.com/" + version
                 regex = re.compile(r"""
                     ^
                     git\+
@@ -148,15 +188,17 @@ class NpmShrinkWrap(FetchMethod):
 
                 url = str(uri)
 
-            # local tarball sources and local link sources are unsupported
             else:
                 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
 
+            # name is needed by unpack tracer for module mapping
             ud.deps.append({
+                "name": name,
                 "url": url,
                 "localpath": localpath,
                 "extrapaths": extrapaths,
                 "destsuffix": destsuffix,
+                "unpack": unpack,
             })
 
         try:
@@ -177,17 +219,23 @@ class NpmShrinkWrap(FetchMethod):
         # This fetcher resolves multiple URIs from a shrinkwrap file and then
         # forwards it to a proxy fetcher. The management of the donestamp file,
         # the lockfile and the checksums are forwarded to the proxy fetcher.
-        ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+        shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
+        if shrinkwrap_urls:
+            ud.proxy = Fetch(shrinkwrap_urls, data)
         ud.needdonestamp = False
 
     @staticmethod
     def _foreach_proxy_method(ud, handle):
         returns = []
-        for proxy_url in ud.proxy.urls:
-            proxy_ud = ud.proxy.ud[proxy_url]
-            proxy_d = ud.proxy.d
-            proxy_ud.setup_localpath(proxy_d)
-            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+        #Check if there are dependencies before try to fetch them
+        if len(ud.deps) > 0:
+            for proxy_url in ud.proxy.urls:
+                proxy_ud = ud.proxy.ud[proxy_url]
+                proxy_d = ud.proxy.d
+                proxy_ud.setup_localpath(proxy_d)
+                lf = lockfile(proxy_ud.lockfile)
+                returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+                unlockfile(lf)
         return returns
 
     def verify_donestamp(self, ud, d):
@@ -224,6 +272,7 @@ class NpmShrinkWrap(FetchMethod):
         destsuffix = ud.parm.get("destsuffix")
         if destsuffix:
             destdir = os.path.join(rootdir, destsuffix)
+        ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
 
         bb.utils.mkdirhier(destdir)
         bb.utils.copyfile(ud.shrinkwrap_file,
@@ -237,7 +286,16 @@ class NpmShrinkWrap(FetchMethod):
 
         for dep in manual:
             depdestdir = os.path.join(destdir, dep["destsuffix"])
-            npm_unpack(dep["localpath"], depdestdir, d)
+            if dep["url"]:
+                npm_unpack(dep["localpath"], depdestdir, d)
+            else:
+                depsrcdir= os.path.join(destdir, dep["localpath"])
+                if dep["unpack"]:
+                    npm_unpack(depsrcdir, depdestdir, d)
+                else:
+                    bb.utils.mkdirhier(depdestdir)
+                    cmd = 'cp -fpPRH "%s/." .' % (depsrcdir)
+                    runfetchcmd(cmd, d, workdir=depdestdir)
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index d9ce44390c..495ac8a30a 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """
@@ -9,6 +11,7 @@ Based on the svn "Fetch" implementation.
 
 import logging
 import os
+import re
 import  bb
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
@@ -36,6 +39,7 @@ class Osc(FetchMethod):
         # Create paths to osc checkouts
         oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
+        ud.oscdir = oscdir
         ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
@@ -43,13 +47,13 @@ class Osc(FetchMethod):
             ud.revision = ud.parm['rev']
         else:
             pv = d.getVar("PV", False)
-            rev = bb.fetch2.srcrev_internal_helper(ud, d)
+            rev = bb.fetch2.srcrev_internal_helper(ud, d, '')
             if rev:
                 ud.revision = rev
             else:
                 ud.revision = ""
 
-        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision))
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision))
 
     def _buildosccommand(self, ud, d, command):
         """
@@ -59,26 +63,49 @@ class Osc(FetchMethod):
 
         basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
-        proto = ud.parm.get('protocol', 'ocs')
+        proto = ud.parm.get('protocol', 'https')
 
         options = []
 
         config = "-c %s" % self.generate_config(ud, d)
 
-        if ud.revision:
+        if getattr(ud, 'revision', ''):
             options.append("-r %s" % ud.revision)
 
         coroot = self._strip_leading_slashes(ud.path)
 
         if command == "fetch":
-            osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options))
+            osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options))
         elif command == "update":
-            osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
+            osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options))
+        elif command == "api_source":
+            osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module)
         else:
             raise FetchError("Invalid osc command %s" % command, ud.url)
 
         return osccmd
 
+    def _latest_revision(self, ud, d, name):
+        """
+        Fetch latest revision for the given package
+        """
+        api_source_cmd = self._buildosccommand(ud, d, "api_source")
+
+        output = runfetchcmd(api_source_cmd, d)
+        match = re.match(r'<directory ?.* rev="(\d+)".*>', output)
+        if match is None:
+            raise FetchError("Unable to parse osc response", ud.url)
+        return match.groups()[0]
+
+    def _revision_key(self, ud, d, name):
+        """
+        Return a unique key for the url
+        """
+        # Collapse adjacent slashes
+        slash_re = re.compile(r"/+")
+        rev = getattr(ud, 'revision', "latest")
+        return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev)
+
     def download(self, ud, d):
         """
         Fetch url
@@ -86,7 +113,7 @@ class Osc(FetchMethod):
 
         logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
+        if os.access(ud.moddir, os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -114,20 +141,23 @@ class Osc(FetchMethod):
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(d.getVar('OSCDIR'), "oscrc")
+        config_path = os.path.join(ud.oscdir, "oscrc")
+        if not os.path.exists(ud.oscdir):
+            bb.utils.mkdirhier(ud.oscdir)
+
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
         f = open(config_path, 'w')
+        proto = ud.parm.get('protocol', 'https')
         f.write("[general]\n")
-        f.write("apisrv = %s\n" % ud.host)
-        f.write("scheme = http\n")
+        f.write("apiurl = %s://%s\n" % (proto, ud.host))
         f.write("su-wrapper = su -c\n")
         f.write("build-root = %s\n" % d.getVar('WORKDIR'))
         f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
-        f.write("[%s]\n" % ud.host)
+        f.write("[%s://%s]\n" % (proto, ud.host))
         f.write("user = %s\n" % ud.parm["user"])
         f.write("pass = %s\n" % ud.parm["pswd"])
         f.close()
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index e2a41a4a12..3b6fa4b1ec 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -134,7 +134,7 @@ class Perforce(FetchMethod):
 
         ud.setup_revisions(d)
 
-        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision))
+        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py
index ffca73c8e4..6b8ffd5359 100644
--- a/bitbake/lib/bb/fetch2/s3.py
+++ b/bitbake/lib/bb/fetch2/s3.py
@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use.
 import os
 import bb
 import urllib.request, urllib.parse, urllib.error
+import re
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import runfetchcmd
 
+def convertToBytes(value, unit):
+    value = float(value)
+    if (unit == "KiB"):
+        value = value*1024.0;
+    elif (unit == "MiB"):
+        value = value*1024.0*1024.0;
+    elif (unit == "GiB"):
+        value = value*1024.0*1024.0*1024.0;
+    return value
+
+class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from s3 cp output, e.g.:
+    Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
+    """
+    def __init__(self, d):
+        super(S3ProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
+        if percs:
+            completed = (percs[-1][0])
+            completedUnit = (percs[-1][1])
+            total = (percs[-1][2])
+            totalUnit = (percs[-1][3])
+            completed = convertToBytes(completed, completedUnit)
+            total = convertToBytes(total, totalUnit)
+            progress = (completed/total)*100.0
+            rate = percs[-1][4]
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class S3(FetchMethod):
     """Class to fetch urls via 'aws s3'"""
 
@@ -52,7 +89,9 @@ class S3(FetchMethod):
 
         cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
         bb.fetch2.check_network_access(d, cmd, ud.url)
-        runfetchcmd(cmd, d)
+
+        progresshandler = S3ProgressHandler(d)
+        runfetchcmd(cmd, d, False, log=progresshandler)
 
         # Additional sanity checks copied from the wget class (although there
         # are no known issues which mean these are required, treat the aws cli
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
index f87f292e5d..7884cce949 100644
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ b/bitbake/lib/bb/fetch2/sftp.py
@@ -103,7 +103,7 @@ class SFTP(FetchMethod):
         if path[:3] == '/~/':
             path = path[3:]
 
-        remote = '%s%s:%s' % (user, urlo.hostname, path)
+        remote = '"%s%s:%s"' % (user, urlo.hostname, path)
 
         cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
 
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 2c8557e1f8..0cbb2a6f25 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -32,6 +32,7 @@ IETF secsh internet draft:
 
 import re, os
 from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
+import urllib
 
 
 __pattern__ = re.compile(r'''
@@ -40,9 +41,9 @@ __pattern__ = re.compile(r'''
  (                   # Optional username/password block
   (?P<user>\S+)      # username
   (:(?P<pass>\S+))?  # colon followed by the password (optional)
- )?
  (?P<cparam>(;[^;]+)*)?  # connection parameters block (optional)
  @
+ )?
  (?P<host>\S+?)          # non-greedy match of the host
  (:(?P<port>[0-9]+))?    # colon followed by the port (optional)
  /
@@ -70,6 +71,7 @@ class SSH(FetchMethod):
                 "git:// prefix with protocol=ssh", urldata.url)
         m = __pattern__.match(urldata.url)
         path = m.group('path')
+        path = urllib.parse.unquote(path)
         host = m.group('host')
         urldata.localpath = os.path.join(d.getVar('DL_DIR'),
                 os.path.basename(os.path.normpath(path)))
@@ -96,6 +98,11 @@ class SSH(FetchMethod):
             fr += '@%s' % host
         else:
             fr = host
+
+        if path[0] != '~':
+            path = '/%s' % path
+        path = urllib.parse.unquote(path)
+
         fr += ':%s' % path
 
         cmd = 'scp -B -r %s %s %s/' % (
@@ -108,3 +115,41 @@ class SSH(FetchMethod):
 
         runfetchcmd(cmd, d)
 
+    def checkstatus(self, fetch, urldata, d):
+        """
+        Check the status of the url
+        """
+        m = __pattern__.match(urldata.url)
+        path = m.group('path')
+        host = m.group('host')
+        port = m.group('port')
+        user = m.group('user')
+        password = m.group('pass')
+
+        if port:
+            portarg = '-P %s' % port
+        else:
+            portarg = ''
+
+        if user:
+            fr = user
+            if password:
+                fr += ':%s' % password
+            fr += '@%s' % host
+        else:
+            fr = host
+
+        if path[0] != '~':
+            path = '/%s' % path
+        path = urllib.parse.unquote(path)
+
+        cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % (
+            portarg,
+            fr,
+            path
+        )
+
+        check_network_access(d, cmd, urldata.url)
+        runfetchcmd(cmd, d)
+
+        return True
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 8856ef1c62..d40e4d2909 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -57,7 +57,12 @@ class Svn(FetchMethod):
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
 
-        ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision))
+        # Whether to use the @REV peg-revision syntax in the svn command or not
+        ud.pegrevision = True
+        if 'nopegrevision' in ud.parm:
+            ud.pegrevision = False
+
+        ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision]))
 
     def _buildsvncommand(self, ud, d, command):
         """
@@ -86,7 +91,7 @@ class Svn(FetchMethod):
         if command == "info":
             svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         elif command == "log1":
-            svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+            svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
 
@@ -98,7 +103,8 @@ class Svn(FetchMethod):
 
             if ud.revision:
                 options.append("-r %s" % ud.revision)
-                suffix = "@%s" % (ud.revision)
+                if ud.pegrevision:
+                    suffix = "@%s" % (ud.revision)
 
             if command == "fetch":
                 transportuser = ud.parm.get("transportuser", "")
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 6d82f3af07..fbfa6938ac 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -26,7 +26,6 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
-from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
@@ -52,18 +51,24 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 
 
 class Wget(FetchMethod):
+    """Class to fetch urls via 'wget'"""
 
     # CDNs like CloudFlare may do a 'browser integrity test' which can fail
     # with the standard wget/urllib User-Agent, so pretend to be a modern
     # browser.
     user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
 
-    """Class to fetch urls via 'wget'"""
+    def check_certs(self, d):
+        """
+        Should certificates be checked?
+        """
+        return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
+
     def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with wget.
         """
-        return ud.type in ['http', 'https', 'ftp']
+        return ud.type in ['http', 'https', 'ftp', 'ftps']
 
     def recommends_checksum(self, urldata):
         return True
@@ -82,7 +87,13 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+
+        if ud.type == 'ftp' or ud.type == 'ftps':
+            self.basecmd += " --passive-ftp"
+
+        if not self.check_certs(d):
+            self.basecmd += " --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet, workdir=None):
 
@@ -97,13 +108,22 @@ class Wget(FetchMethod):
 
         fetchcmd = self.basecmd
 
-        if 'downloadfilename' in ud.parm:
-            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
-            bb.utils.mkdirhier(os.path.dirname(localpath))
-            fetchcmd += " -O %s" % shlex.quote(localpath)
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
-            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+            fetchcmd += " --auth-no-challenge"
+            if ud.parm.get("redirectauth", "1") == "1":
+                # An undocumented feature of wget is that if the
+                # username/password are specified on the URI, wget will only
+                # send the Authorization header to the first host and not to
+                # any hosts that it is redirected to.  With the increasing
+                # usage of temporary AWS URLs, this difference now matters as
+                # AWS will reject any request that has authentication both in
+                # the query parameters (from the redirect) and in the
+                # Authorization header.
+                fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
 
         uri = ud.url.split(";")[0]
         if os.path.exists(ud.localpath):
@@ -114,6 +134,15 @@ class Wget(FetchMethod):
 
         self._runwget(ud, d, fetchcmd, False)
 
+        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
+        # original file, which might be a race (imagine two recipes referencing the same
+        # source, one with an incorrect checksum)
+        bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
+
+        # Remove the ".tmp" and move the file into position atomically
+        # Our lock prevents multiple writers but mirroring code may grab incomplete files
+        os.rename(localpath, localpath[:-4])
+
         # Sanity check since wget can pretend it succeed when it didn't
         # Also, this used to happen if sourceforge sent us to the mirror page
         if not os.path.exists(ud.localpath):
@@ -209,7 +238,7 @@ class Wget(FetchMethod):
                         # We let the request fail and expect it to be
                         # tried once more ("try_again" in check_status()),
                         # with the dead connection removed from the cache.
-                        # If it still fails, we give up, which can happend for bad
+                        # If it still fails, we give up, which can happen for bad
                         # HTTP proxy settings.
                         fetch.connection_cache.remove_connection(h.host, h.port)
                     raise urllib.error.URLError(err)
@@ -282,64 +311,76 @@ class Wget(FetchMethod):
                 newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
                 newreq.get_method = req.get_method
                 return newreq
-        exported_proxies = export_proxies(d)
-
-        handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
-        if exported_proxies:
-            handlers.append(urllib.request.ProxyHandler())
-        handlers.append(CacheHTTPHandler())
-        # Since Python 2.7.9 ssl cert validation is enabled by default
-        # see PEP-0476, this causes verification errors on some https servers
-        # so disable by default.
-        import ssl
-        if hasattr(ssl, '_create_unverified_context'):
-            handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
-        opener = urllib.request.build_opener(*handlers)
-
-        try:
-            uri = ud.url.split(";")[0]
-            r = urllib.request.Request(uri)
-            r.get_method = lambda: "HEAD"
-            # Some servers (FusionForge, as used on Alioth) require that the
-            # optional Accept header is set.
-            r.add_header("Accept", "*/*")
-            r.add_header("User-Agent", self.user_agent)
-            def add_basic_auth(login_str, request):
-                '''Adds Basic auth to http request, pass in login:password as string'''
-                import base64
-                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
-                authheader = "Basic %s" % encodeuser
-                r.add_header("Authorization", authheader)
-
-            if ud.user and ud.pswd:
-                add_basic_auth(ud.user + ':' + ud.pswd, r)
 
-            try:
-                import netrc
-                n = netrc.netrc()
-                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
-                add_basic_auth("%s:%s" % (login, password), r)
-            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
-                pass
-
-            with opener.open(r) as response:
-                pass
-        except urllib.error.URLError as e:
-            if try_again:
-                logger.debug2("checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
+        # We need to update the environment here as both the proxy and HTTPS
+        # handlers need variables set. The proxy needs http_proxy and friends to
+        # be set, and HTTPSHandler ends up calling into openssl to load the
+        # certificates. In buildtools configurations this will be looking at the
+        # wrong place for certificates by default: we set SSL_CERT_FILE to the
+        # right location in the buildtools environment script but as BitBake
+        # prunes prunes the environment this is lost. When binaries are executed
+        # runfetchcmd ensures these values are in the environment, but this is
+        # pure Python so we need to update the environment.
+        #
+        # Avoid tramping the environment too much by using bb.utils.environment
+        # to scope the changes to the build_opener request, which is when the
+        # environment lookups happen.
+        newenv = bb.fetch2.get_fetcher_environment(d)
+
+        with bb.utils.environment(**newenv):
+            import ssl
+
+            if self.check_certs(d):
+                context = ssl.create_default_context()
             else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug2("checkstatus() urlopen failed: %s" % e)
-                return False
-        except ConnectionResetError as e:
-            if try_again:
-                logger.debug2("checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
-            else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug2("checkstatus() urlopen failed: %s" % e)
-                return False
+                context = ssl._create_unverified_context()
+
+            handlers = [FixedHTTPRedirectHandler,
+                        HTTPMethodFallback,
+                        urllib.request.ProxyHandler(),
+                        CacheHTTPHandler(),
+                        urllib.request.HTTPSHandler(context=context)]
+            opener = urllib.request.build_opener(*handlers)
+
+            try:
+                uri_base = ud.url.split(";")[0]
+                uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
+                r = urllib.request.Request(uri)
+                r.get_method = lambda: "HEAD"
+                # Some servers (FusionForge, as used on Alioth) require that the
+                # optional Accept header is set.
+                r.add_header("Accept", "*/*")
+                r.add_header("User-Agent", self.user_agent)
+                def add_basic_auth(login_str, request):
+                    '''Adds Basic auth to http request, pass in login:password as string'''
+                    import base64
+                    encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
+                    authheader = "Basic %s" % encodeuser
+                    r.add_header("Authorization", authheader)
+
+                if ud.user and ud.pswd:
+                    add_basic_auth(ud.user + ':' + ud.pswd, r)
+
+                try:
+                    import netrc
+                    auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
+                    if auth_data:
+                        login, _, password = auth_data
+                        add_basic_auth("%s:%s" % (login, password), r)
+                except (FileNotFoundError, netrc.NetrcParseError):
+                    pass
+
+                with opener.open(r, timeout=30) as response:
+                    pass
+            except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
+                if try_again:
+                    logger.debug2("checkstatus: trying again")
+                    return self.checkstatus(fetch, ud, d, False)
+                else:
+                    # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                    logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
+                    return False
+
         return True
 
     def _parse_path(self, regex, s):
@@ -472,7 +513,7 @@ class Wget(FetchMethod):
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group('ver')
@@ -548,7 +589,7 @@ class Wget(FetchMethod):
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
 
         # match name, version and archive type of a package
         package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
@@ -599,10 +640,10 @@ class Wget(FetchMethod):
             # search for version matches on folders inside the path, like:
             # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
             dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
-            m = dirver_regex.search(path)
+            m = dirver_regex.findall(path)
             if m:
                 pn = d.getVar('PN')
-                dirver = m.group('dirver')
+                dirver = m[-1][0]
 
                 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
                 if not dirver_pn_regex.search(dirver):
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py
index 06bad495ac..bca8ebfa09 100755
--- a/bitbake/lib/bb/main.py
+++ b/bitbake/lib/bb/main.py
@@ -12,11 +12,12 @@
 import os
 import sys
 import logging
-import optparse
+import argparse
 import warnings
 import fcntl
 import time
 import traceback
+import datetime
 
 import bb
 from bb import event
@@ -43,18 +44,18 @@ def present_options(optionlist):
     else:
         return optionlist[0]
 
-class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
-    def format_option(self, option):
+class BitbakeHelpFormatter(argparse.HelpFormatter):
+    def _get_help_string(self, action):
         # We need to do this here rather than in the text we supply to
         # add_option() because we don't want to call list_extension_modules()
         # on every execution (since it imports all of the modules)
         # Note also that we modify option.help rather than the returned text
         # - this is so that we don't have to re-format the text ourselves
-        if option.dest == 'ui':
+        if action.dest == 'ui':
             valid_uis = list_extension_modules(bb.ui, 'main')
-            option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
+            return action.help.replace('@CHOICES@', present_options(valid_uis))
 
-        return optparse.IndentedHelpFormatter.format_option(self, option)
+        return action.help
 
 def list_extension_modules(pkg, checkattr):
     """
@@ -112,189 +113,209 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
         warnlog.warning(s)
 
 warnings.showwarning = _showwarning
-warnings.filterwarnings("ignore")
-warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)")
-warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
-warnings.filterwarnings("ignore", category=ImportWarning)
-warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
-warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
-
 
 def create_bitbake_parser():
-    parser = optparse.OptionParser(
-        formatter=BitbakeHelpFormatter(),
-        version="BitBake Build Tool Core version %s" % bb.__version__,
-        usage="""%prog [options] [recipename/target recipe:do_task ...]
-
-    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
-    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-    will provide the layer, BBFILES and other configuration information.""")
-
-    parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
-                      help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
-                           "not handle any dependencies from other recipes.")
-
-    parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
-                      help="Continue as much as possible after an error. While the target that "
-                           "failed and anything depending on it cannot be built, as much as "
-                           "possible will be built before stopping.")
-
-    parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
-                      help="Force the specified targets/task to run (invalidating any "
-                           "existing stamp file).")
-
-    parser.add_option("-c", "--cmd", action="store", dest="cmd",
-                      help="Specify the task to execute. The exact options available "
-                           "depend on the metadata. Some examples might be 'compile'"
-                           " or 'populate_sysroot' or 'listtasks' may give a list of "
-                           "the tasks available.")
-
-    parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
-                      help="Invalidate the stamp for the specified task such as 'compile' "
-                           "and then run the default task for the specified target(s).")
-
-    parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
-                      help="Read the specified file before bitbake.conf.")
-
-    parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
-                      help="Read the specified file after bitbake.conf.")
-
-    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
-                      help="Enable tracing of shell tasks (with 'set -x'). "
-                           "Also print bb.note(...) messages to stdout (in "
-                           "addition to writing them to ${T}/log.do_<task>).")
-
-    parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
-                      help="Increase the debug level. You can specify this "
-                           "more than once. -D sets the debug level to 1, "
-                           "where only bb.debug(1, ...) messages are printed "
-                           "to stdout; -DD sets the debug level to 2, where "
-                           "both bb.debug(1, ...) and bb.debug(2, ...) "
-                           "messages are printed; etc. Without -D, no debug "
-                           "messages are printed. Note that -D only affects "
-                           "output to stdout. All debug messages are written "
-                           "to ${T}/log.do_taskname, regardless of the debug "
-                           "level.")
-
-    parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
-                      help="Output less log message data to the terminal. You can specify this more than once.")
-
-    parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
-                      help="Don't execute, just go through the motions.")
-
-    parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
-                      default=[], metavar="SIGNATURE_HANDLER",
-                      help="Dump out the signature construction information, with no task "
-                           "execution. The SIGNATURE_HANDLER parameter is passed to the "
-                           "handler. Two common values are none and printdiff but the handler "
-                           "may define more/less. none means only dump the signature, printdiff"
-                           " means compare the dumped signature with the cached one.")
-
-    parser.add_option("-p", "--parse-only", action="store_true",
-                      dest="parse_only", default=False,
-                      help="Quit after parsing the BB recipes.")
-
-    parser.add_option("-s", "--show-versions", action="store_true",
-                      dest="show_versions", default=False,
-                      help="Show current and preferred versions of all recipes.")
-
-    parser.add_option("-e", "--environment", action="store_true",
-                      dest="show_environment", default=False,
-                      help="Show the global or per-recipe environment complete with information"
-                           " about where variables were set/changed.")
-
-    parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
-                      help="Save dependency tree information for the specified "
-                           "targets in the dot syntax.")
-
-    parser.add_option("-I", "--ignore-deps", action="append",
-                      dest="extra_assume_provided", default=[],
-                      help="Assume these dependencies don't exist and are already provided "
-                           "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
-                           "graphs more appealing")
-
-    parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
-                      help="Show debug logging for the specified logging domains")
-
-    parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
-                      help="Profile the command and save reports.")
+    parser = argparse.ArgumentParser(
+        description="""\
+            It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+            will provide the layer, BBFILES and other configuration information.
+            """,
+        formatter_class=BitbakeHelpFormatter,
+        allow_abbrev=False,
+        add_help=False, # help is manually added below in a specific argument group
+    )
+
+    general_group  = parser.add_argument_group('General options')
+    task_group     = parser.add_argument_group('Task control options')
+    exec_group     = parser.add_argument_group('Execution control options')
+    logging_group  = parser.add_argument_group('Logging/output control options')
+    server_group   = parser.add_argument_group('Server options')
+    config_group   = parser.add_argument_group('Configuration options')
+
+    general_group.add_argument("targets", nargs="*", metavar="recipename/target",
+                        help="Execute the specified task (default is 'build') for these target "
+                             "recipes (.bb files).")
+
+    general_group.add_argument("-s", "--show-versions", action="store_true",
+                        help="Show current and preferred versions of all recipes.")
+
+    general_group.add_argument("-e", "--environment", action="store_true",
+                        dest="show_environment",
+                        help="Show the global or per-recipe environment complete with information"
+                             " about where variables were set/changed.")
+
+    general_group.add_argument("-g", "--graphviz", action="store_true", dest="dot_graph",
+                        help="Save dependency tree information for the specified "
+                             "targets in the dot syntax.")
 
     # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-    parser.add_option("-u", "--ui", action="store", dest="ui",
-                      default=os.environ.get('BITBAKE_UI', 'knotty'),
-                      help="The user interface to use (@CHOICES@ - default %default).")
-
-    parser.add_option("", "--token", action="store", dest="xmlrpctoken",
-                      default=os.environ.get("BBTOKEN"),
-                      help="Specify the connection token to be used when connecting "
-                           "to a remote server.")
-
-    parser.add_option("", "--revisions-changed", action="store_true",
-                      dest="revisions_changed", default=False,
-                      help="Set the exit code depending on whether upstream floating "
-                           "revisions have changed or not.")
-
-    parser.add_option("", "--server-only", action="store_true",
-                      dest="server_only", default=False,
-                      help="Run bitbake without a UI, only starting a server "
-                           "(cooker) process.")
-
-    parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
-                      help="The name/address for the bitbake xmlrpc server to bind to.")
-
-    parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
-                      default=os.getenv("BB_SERVER_TIMEOUT"),
-                      help="Set timeout to unload bitbake server due to inactivity, "
-                           "set to -1 means no unload, "
-                           "default: Environment variable BB_SERVER_TIMEOUT.")
-
-    parser.add_option("", "--no-setscene", action="store_true",
-                      dest="nosetscene", default=False,
-                      help="Do not run any setscene tasks. sstate will be ignored and "
-                           "everything needed, built.")
-
-    parser.add_option("", "--skip-setscene", action="store_true",
-                      dest="skipsetscene", default=False,
-                      help="Skip setscene tasks if they would be executed. Tasks previously "
-                           "restored from sstate will be kept, unlike --no-setscene")
-
-    parser.add_option("", "--setscene-only", action="store_true",
-                      dest="setsceneonly", default=False,
-                      help="Only run setscene tasks, don't run any real tasks.")
-
-    parser.add_option("", "--remote-server", action="store", dest="remote_server",
-                      default=os.environ.get("BBSERVER"),
-                      help="Connect to the specified server.")
-
-    parser.add_option("-m", "--kill-server", action="store_true",
-                      dest="kill_server", default=False,
-                      help="Terminate any running bitbake server.")
-
-    parser.add_option("", "--observe-only", action="store_true",
-                      dest="observe_only", default=False,
-                      help="Connect to a server as an observing-only client.")
-
-    parser.add_option("", "--status-only", action="store_true",
-                      dest="status_only", default=False,
-                      help="Check the status of the remote bitbake server.")
-
-    parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
-                      default=os.environ.get("BBEVENTLOG"),
-                      help="Writes the event log of the build to a bitbake event json file. "
-                           "Use '' (empty string) to assign the name automatically.")
-
-    parser.add_option("", "--runall", action="append", dest="runall",
-                      help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
-
-    parser.add_option("", "--runonly", action="append", dest="runonly",
-                      help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    general_group.add_argument("-u", "--ui",
+                        default=os.environ.get('BITBAKE_UI', 'knotty'),
+                        help="The user interface to use (@CHOICES@ - default %(default)s).")
+
+    general_group.add_argument("--version", action="store_true",
+                        help="Show programs version and exit.")
+
+    general_group.add_argument('-h', '--help', action='help',
+                        help='Show this help message and exit.')
+
+
+    task_group.add_argument("-f", "--force", action="store_true",
+                        help="Force the specified targets/task to run (invalidating any "
+                             "existing stamp file).")
+
+    task_group.add_argument("-c", "--cmd",
+                        help="Specify the task to execute. The exact options available "
+                             "depend on the metadata. Some examples might be 'compile'"
+                             " or 'populate_sysroot' or 'listtasks' may give a list of "
+                             "the tasks available.")
+
+    task_group.add_argument("-C", "--clear-stamp", dest="invalidate_stamp",
+                        help="Invalidate the stamp for the specified task such as 'compile' "
+                             "and then run the default task for the specified target(s).")
+
+    task_group.add_argument("--runall", action="append", default=[],
+                        help="Run the specified task for any recipe in the taskgraph of the "
+                             "specified target (even if it wouldn't otherwise have run).")
+
+    task_group.add_argument("--runonly", action="append",
+                        help="Run only the specified task within the taskgraph of the "
+                             "specified targets (and any task dependencies those tasks may have).")
+
+    task_group.add_argument("--no-setscene", action="store_true",
+                        dest="nosetscene",
+                        help="Do not run any setscene tasks. sstate will be ignored and "
+                            "everything needed, built.")
+
+    task_group.add_argument("--skip-setscene", action="store_true",
+                        dest="skipsetscene",
+                        help="Skip setscene tasks if they would be executed. Tasks previously "
+                            "restored from sstate will be kept, unlike --no-setscene.")
+
+    task_group.add_argument("--setscene-only", action="store_true",
+                        dest="setsceneonly",
+                        help="Only run setscene tasks, don't run any real tasks.")
+
+
+    exec_group.add_argument("-n", "--dry-run", action="store_true",
+                        help="Don't execute, just go through the motions.")
+
+    exec_group.add_argument("-p", "--parse-only", action="store_true",
+                        help="Quit after parsing the BB recipes.")
+
+    exec_group.add_argument("-k", "--continue", action="store_false", dest="halt",
+                        help="Continue as much as possible after an error. While the target that "
+                             "failed and anything depending on it cannot be built, as much as "
+                             "possible will be built before stopping.")
+
+    exec_group.add_argument("-P", "--profile", action="store_true",
+                        help="Profile the command and save reports.")
+
+    exec_group.add_argument("-S", "--dump-signatures", action="append",
+                        default=[], metavar="SIGNATURE_HANDLER",
+                        help="Dump out the signature construction information, with no task "
+                             "execution. The SIGNATURE_HANDLER parameter is passed to the "
+                             "handler. Two common values are none and printdiff but the handler "
+                             "may define more/less. none means only dump the signature, printdiff"
+                             " means recursively compare the dumped signature with the most recent"
+                             " one in a local build or sstate cache (can be used to find out why tasks re-run"
+                             " when that is not expected)")
+
+    exec_group.add_argument("--revisions-changed", action="store_true",
+                        help="Set the exit code depending on whether upstream floating "
+                            "revisions have changed or not.")
+
+    exec_group.add_argument("-b", "--buildfile",
+                        help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
+                             "not handle any dependencies from other recipes.")
+
+    logging_group.add_argument("-D", "--debug", action="count", default=0,
+                        help="Increase the debug level. You can specify this "
+                             "more than once. -D sets the debug level to 1, "
+                             "where only bb.debug(1, ...) messages are printed "
+                             "to stdout; -DD sets the debug level to 2, where "
+                             "both bb.debug(1, ...) and bb.debug(2, ...) "
+                             "messages are printed; etc. Without -D, no debug "
+                             "messages are printed. Note that -D only affects "
+                             "output to stdout. All debug messages are written "
+                             "to ${T}/log.do_taskname, regardless of the debug "
+                             "level.")
+
+    logging_group.add_argument("-l", "--log-domains", action="append", dest="debug_domains",
+                        default=[],
+                        help="Show debug logging for the specified logging domains.")
+
+    logging_group.add_argument("-v", "--verbose", action="store_true",
+                        help="Enable tracing of shell tasks (with 'set -x'). "
+                             "Also print bb.note(...) messages to stdout (in "
+                             "addition to writing them to ${T}/log.do_<task>).")
+
+    logging_group.add_argument("-q", "--quiet", action="count", default=0,
+                        help="Output less log message data to the terminal. You can specify this "
+                             "more than once.")
+
+    logging_group.add_argument("-w", "--write-log", dest="writeeventlog",
+                        default=os.environ.get("BBEVENTLOG"),
+                        help="Writes the event log of the build to a bitbake event json file. "
+                            "Use '' (empty string) to assign the name automatically.")
+
+
+    server_group.add_argument("-B", "--bind", default=False,
+                        help="The name/address for the bitbake xmlrpc server to bind to.")
+
+    server_group.add_argument("-T", "--idle-timeout", type=float, dest="server_timeout",
+                        default=os.getenv("BB_SERVER_TIMEOUT"),
+                        help="Set timeout to unload bitbake server due to inactivity, "
+                             "set to -1 means no unload, "
+                             "default: Environment variable BB_SERVER_TIMEOUT.")
+
+    server_group.add_argument("--remote-server",
+                        default=os.environ.get("BBSERVER"),
+                        help="Connect to the specified server.")
+
+    server_group.add_argument("-m", "--kill-server", action="store_true",
+                        help="Terminate any running bitbake server.")
+
+    server_group.add_argument("--token", dest="xmlrpctoken",
+                        default=os.environ.get("BBTOKEN"),
+                        help="Specify the connection token to be used when connecting "
+                             "to a remote server.")
+
+    server_group.add_argument("--observe-only", action="store_true",
+                        help="Connect to a server as an observing-only client.")
+
+    server_group.add_argument("--status-only", action="store_true",
+                        help="Check the status of the remote bitbake server.")
+
+    server_group.add_argument("--server-only", action="store_true",
+                        help="Run bitbake without a UI, only starting a server "
+                            "(cooker) process.")
+
+
+    config_group.add_argument("-r", "--read", action="append", dest="prefile", default=[],
+                        help="Read the specified file before bitbake.conf.")
+
+    config_group.add_argument("-R", "--postread", action="append", dest="postfile", default=[],
+                        help="Read the specified file after bitbake.conf.")
+
+
+    config_group.add_argument("-I", "--ignore-deps", action="append",
+                        dest="extra_assume_provided", default=[],
+                        help="Assume these dependencies don't exist and are already provided "
+                             "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
+                             "graphs more appealing.")
+
     return parser
 
 
 class BitBakeConfigParameters(cookerdata.ConfigParameters):
     def parseCommandLine(self, argv=sys.argv):
         parser = create_bitbake_parser()
-        options, targets = parser.parse_args(argv)
+        options = parser.parse_intermixed_args(argv[1:])
+
+        if options.version:
+            print("BitBake Build Tool Core version %s" % bb.__version__)
+            sys.exit(0)
 
         if options.quiet and options.verbose:
             parser.error("options --quiet and --verbose are mutually exclusive")
@@ -326,7 +347,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         else:
             options.xmlrpcinterface = (None, 0)
 
-        return options, targets[1:]
+        return options, options.targets
 
 
 def bitbake_main(configParams, configuration):
@@ -391,6 +412,9 @@ def bitbake_main(configParams, configuration):
 
     return 1
 
+def timestamp():
+    return datetime.datetime.now().strftime('%H:%M:%S.%f')
+
 def setup_bitbake(configParams, extrafeatures=None):
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
@@ -398,6 +422,11 @@ def setup_bitbake(configParams, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
+    if configParams.dump_signatures:
+        if extrafeatures is None:
+            extrafeatures = []
+        extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO)
+
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -425,7 +454,7 @@ def setup_bitbake(configParams, extrafeatures=None):
         retries = 8
         while retries:
             try:
-                topdir, lock = lockBitbake()
+                topdir, lock, lockfile = lockBitbake()
                 sockname = topdir + "/bitbake.sock"
                 if lock:
                     if configParams.status_only or configParams.kill_server:
@@ -436,18 +465,22 @@ def setup_bitbake(configParams, extrafeatures=None):
                     logger.info("Starting bitbake server...")
                     # Clear the event queue since we already displayed messages
                     bb.event.ui_queue = []
-                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface)
+                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface, configParams.profile)
 
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry... (%s)" % timestamp())
+                        procs = bb.server.process.get_lockfile_process_msg(lockfile)
+                        if procs:
+                            logger.info("Processes holding bitbake.lock (missing socket %s):\n%s" % (sockname, procs))
+                        logger.info("Directory listing: %s" % (str(os.listdir(topdir))))
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock
                         while not lock and i < 50:
                             time.sleep(0.1)
-                            _, lock = lockBitbake()
+                            _, lock, _ = lockBitbake()
                             i += 1
                         if lock:
                             bb.utils.unlockfile(lock)
@@ -466,9 +499,9 @@ def setup_bitbake(configParams, extrafeatures=None):
                 retries -= 1
                 tryno = 8 - retries
                 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)):
-                    logger.info("Retrying server connection (#%d)..." % tryno)
+                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, timestamp()))
                 else:
-                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
+                    logger.info("Retrying server connection (#%d)... (%s, %s)" % (tryno, traceback.format_exc(), timestamp()))
 
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
@@ -497,5 +530,5 @@ def lockBitbake():
         bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?")
         raise BBMainFatal
     lockfile = topdir + "/bitbake.lock"
-    return topdir, bb.utils.lockfile(lockfile, False, False)
+    return topdir, bb.utils.lockfile(lockfile, False, False), lockfile
 
diff --git a/bitbake/lib/bb/monitordisk.py b/bitbake/lib/bb/monitordisk.py
index 98f2109ed2..f928210351 100644
--- a/bitbake/lib/bb/monitordisk.py
+++ b/bitbake/lib/bb/monitordisk.py
@@ -76,7 +76,12 @@ def getDiskData(BBDirs):
             return None
 
         action = pathSpaceInodeRe.group(1)
-        if action not in ("ABORT", "STOPTASKS", "WARN"):
+        if action == "ABORT":
+            # Emit a deprecation warning
+            logger.warnonce("The BB_DISKMON_DIRS \"ABORT\" action has been renamed to \"HALT\", update configuration")
+            action = "HALT"
+
+        if action not in ("HALT", "STOPTASKS", "WARN"):
             printErr("Unknown disk space monitor action: %s" % action)
             return None
 
@@ -177,7 +182,7 @@ class diskMonitor:
                     # use them to avoid printing too many warning messages
                     self.preFreeS = {}
                     self.preFreeI = {}
-                    # This is for STOPTASKS and ABORT, to avoid printing the message
+                    # This is for STOPTASKS and HALT, to avoid printing the message
                     # repeatedly while waiting for the tasks to finish
                     self.checked = {}
                     for k in self.devDict:
@@ -219,8 +224,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
-                    elif action == "ABORT" and not self.checked[k]:
-                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                    elif action == "HALT" and not self.checked[k]:
+                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
@@ -229,9 +234,10 @@ class diskMonitor:
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
-                    # Some filesystems use dynamic inodes so can't run out
-                    # (e.g. btrfs). This is reported by the inode count being 0.
-                    if st.f_files == 0:
+                    # Some filesystems use dynamic inodes so can't run out.
+                    # This is reported by the inode count being 0 (btrfs) or the free
+                    # inode count being -1 (cephfs).
+                    if st.f_files == 0 or st.f_favail == -1:
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN
@@ -245,8 +251,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
-                    elif action  == "ABORT" and not self.checked[k]:
-                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                    elif action  == "HALT" and not self.checked[k]:
+                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index 291b38ff7f..3e18596faa 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -30,7 +30,9 @@ class BBLogFormatter(logging.Formatter):
     PLAIN = logging.INFO + 1
     VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
+    ERRORONCE = logging.ERROR - 1
     WARNING = logging.WARNING
+    WARNONCE = logging.WARNING - 1
     CRITICAL = logging.CRITICAL
 
     levelnames = {
@@ -42,7 +44,9 @@ class BBLogFormatter(logging.Formatter):
         PLAIN  : '',
         VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
+        WARNONCE : 'WARNING',
         ERROR   : 'ERROR',
+        ERRORONCE   : 'ERROR',
         CRITICAL: 'ERROR',
     }
 
@@ -58,7 +62,9 @@ class BBLogFormatter(logging.Formatter):
         PLAIN   : BASECOLOR,
         VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
+        WARNONCE : YELLOW,
         ERROR   : RED,
+        ERRORONCE : RED,
         CRITICAL: RED,
     }
 
@@ -121,6 +127,22 @@ class BBLogFilter(object):
             return True
         return False
 
+class LogFilterShowOnce(logging.Filter):
+    def __init__(self):
+        self.seen_warnings = set()
+        self.seen_errors = set()
+
+    def filter(self, record):
+        if record.levelno == bb.msg.BBLogFormatter.WARNONCE:
+            if record.msg in self.seen_warnings:
+                return False
+            self.seen_warnings.add(record.msg)
+        if record.levelno == bb.msg.BBLogFormatter.ERRORONCE:
+            if record.msg in self.seen_errors:
+                return False
+            self.seen_errors.add(record.msg)
+        return True
+
 class LogFilterGEQLevel(logging.Filter):
     def __init__(self, level):
         self.strlevel = str(level)
@@ -206,8 +228,9 @@ def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers
     """Standalone logger creation function"""
     logger = logging.getLogger(name)
     console = logging.StreamHandler(output)
+    console.addFilter(bb.msg.LogFilterShowOnce())
     format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    if color == 'always' or (color == 'auto' and output.isatty()):
+    if color == 'always' or (color == 'auto' and output.isatty() and os.environ.get('NO_COLOR', '') == ''):
         format.enable_color()
     console.setFormatter(format)
     if preserve_handlers:
@@ -293,10 +316,17 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
 
     # Convert all level parameters to integers in case users want to use the
     # bitbake defined level names
-    for h in logconfig["handlers"].values():
+    for name, h in logconfig["handlers"].items():
         if "level" in h:
             h["level"] = bb.msg.stringToLevel(h["level"])
 
+        # Every handler needs its own instance of the once filter.
+        once_filter_name = name + ".showonceFilter"
+        logconfig.setdefault("filters", {})[once_filter_name] = {
+            "()": "bb.msg.LogFilterShowOnce",
+        }
+        h.setdefault("filters", []).append(once_filter_name)
+
     for l in logconfig["loggers"].values():
         if "level" in l:
             l["level"] = bb.msg.stringToLevel(l["level"])
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index c01807ba87..a4358f1374 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -60,6 +60,14 @@ def cached_mtime_noerror(f):
             return 0
     return __mtime_cache[f]
 
+def check_mtime(f, mtime):
+    try:
+        current_mtime = os.stat(f)[stat.ST_MTIME]
+        __mtime_cache[f] = current_mtime
+    except OSError:
+        current_mtime = 0
+    return current_mtime == mtime
+
 def update_mtime(f):
     try:
         __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
@@ -99,12 +107,12 @@ def supports(fn, data):
             return 1
     return 0
 
-def handle(fn, data, include = 0):
+def handle(fn, data, include=0, baseconfig=False):
     """Call the handler that is appropriate for this file"""
     for h in handlers:
         if h['supports'](fn, data):
             with data.inchistory.include(fn):
-                return h['handle'](fn, data, include)
+                return h['handle'](fn, data, include, baseconfig)
     raise ParseError("not a BitBake file", fn)
 
 def init(fn, data):
@@ -113,6 +121,8 @@ def init(fn, data):
             return h['init'](data)
 
 def init_parser(d):
+    if hasattr(bb.parse, "siggen"):
+        bb.parse.siggen.exit()
     bb.parse.siggen = bb.siggen.init(d)
 
 def resolve_file(fn, d):
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index 50a88f7da7..7581d003fd 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -9,6 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import sys
 import bb
 from bb import methodpool
 from bb.parse import logger
@@ -130,6 +131,10 @@ class DataNode(AstNode):
         else:
             val = groupd["value"]
 
+        if ":append" in key or ":remove" in key or ":prepend" in key:
+            if op in ["append", "prepend", "postdot", "predot", "ques"]:
+                bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.")
+
         flag = None
         if 'flag' in groupd and groupd['flag'] is not None:
             flag = groupd['flag']
@@ -145,7 +150,7 @@ class DataNode(AstNode):
             data.setVar(key, val, parsing=True, **loginfo)
 
 class MethodNode(AstNode):
-    tr_tbl = str.maketrans('/.+-@%&', '_______')
+    tr_tbl = str.maketrans('/.+-@%&~', '________')
 
     def __init__(self, filename, lineno, func_name, body, python, fakeroot):
         AstNode.__init__(self, filename, lineno)
@@ -206,10 +211,12 @@ class ExportFuncsNode(AstNode):
 
     def eval(self, data):
 
+        sentinel = "    # Export function set\n"
         for func in self.n:
             calledfunc = self.classname + "_" + func
 
-            if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
+            basevar = data.getVar(func, False)
+            if basevar and sentinel not in basevar:
                 continue
 
             if data.getVar(func, False):
@@ -219,19 +226,18 @@ class ExportFuncsNode(AstNode):
             for flag in [ "func", "python" ]:
                 if data.getVarFlag(calledfunc, flag, False):
                     data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False))
-            for flag in [ "dirs" ]:
+            for flag in ["dirs", "cleandirs", "fakeroot"]:
                 if data.getVarFlag(func, flag, False):
                     data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False))
             data.setVarFlag(func, "filename", "autogenerated")
             data.setVarFlag(func, "lineno", 1)
 
             if data.getVarFlag(calledfunc, "python", False):
-                data.setVar(func, "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
+                data.setVar(func, sentinel + "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
             else:
                 if "-" in self.classname:
                    bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
-                data.setVar(func, "    " + calledfunc + "\n", parsing=True)
-            data.setVarFlag(func, 'export_func', '1')
+                data.setVar(func, sentinel + "    " + calledfunc + "\n", parsing=True)
 
 class AddTaskNode(AstNode):
     def __init__(self, filename, lineno, func, before, after):
@@ -265,6 +271,41 @@ class BBHandlerNode(AstNode):
             data.setVarFlag(h, "handler", 1)
         data.setVar('__BBHANDLERS', bbhands)
 
+class PyLibNode(AstNode):
+    def __init__(self, filename, lineno, libdir, namespace):
+        AstNode.__init__(self, filename, lineno)
+        self.libdir = libdir
+        self.namespace = namespace
+
+    def eval(self, data):
+        global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split()
+        for m in global_mods:
+            if m not in bb.utils._context:
+                bb.utils._context[m] = __import__(m)
+
+        libdir = data.expand(self.libdir)
+        if libdir not in sys.path:
+            sys.path.append(libdir)
+        try:
+            bb.utils._context[self.namespace] = __import__(self.namespace)
+            toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", [])
+            for i in toimport:
+                bb.utils._context[self.namespace] = __import__(self.namespace + "." + i)
+                mod = getattr(bb.utils._context[self.namespace], i)
+                fn = getattr(mod, "__file__")
+                funcs = {}
+                for f in dir(mod):
+                    if f.startswith("_"):
+                        continue
+                    fcall = getattr(mod, f)
+                    if not callable(fcall):
+                        continue
+                    funcs[f] = fcall
+                bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i))
+
+        except AttributeError as e:
+            bb.error("Error importing OE modules: %s" % str(e))
+
 class InheritNode(AstNode):
     def __init__(self, filename, lineno, classes):
         AstNode.__init__(self, filename, lineno)
@@ -273,6 +314,16 @@ class InheritNode(AstNode):
     def eval(self, data):
         bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data)
 
+class InheritDeferredNode(AstNode):
+    def __init__(self, filename, lineno, classes):
+        AstNode.__init__(self, filename, lineno)
+        self.inherit = (classes, filename, lineno)
+
+    def eval(self, data):
+        inherits = data.getVar('__BBDEFINHERITS', False) or []
+        inherits.append(self.inherit)
+        data.setVar('__BBDEFINHERITS', inherits)
+
 def handleInclude(statements, filename, lineno, m, force):
     statements.append(IncludeNode(filename, lineno, m.group(1), force))
 
@@ -316,10 +367,17 @@ def handleDelTask(statements, filename, lineno, m):
 def handleBBHandlers(statements, filename, lineno, m):
     statements.append(BBHandlerNode(filename, lineno, m.group(1)))
 
+def handlePyLib(statements, filename, lineno, m):
+    statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2)))
+
 def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
 
+def handleInheritDeferred(statements, filename, lineno, m):
+    classes = m.group(1)
+    statements.append(InheritDeferredNode(filename, lineno, classes))
+
 def runAnonFuncs(d):
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
@@ -329,6 +387,10 @@ def runAnonFuncs(d):
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
     try:
+        # Found renamed variables. Exit immediately
+        if d.getVar("_FAILPARSINGERRORHANDLED", False) == True:
+            raise bb.BBHandledException()
+
         for var in d.getVar('__BBHANDLERS', False) or []:
             # try to add the handler
             handlerfn = d.getVarFlag(var, "filename", False)
@@ -353,6 +415,9 @@ def finalize(fn, d, variant = None):
 
         d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
 
+        if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"):
+            bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.")
+
         bb.event.fire(bb.event.RecipeParsed(fn), d)
     finally:
         bb.event.set_handlers(saved_handlers)
@@ -379,6 +444,14 @@ def multi_finalize(fn, d):
         logger.debug("Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
 
+    while True:
+        inherits = d.getVar('__BBDEFINHERITS', False) or []
+        if not inherits:
+            break
+        inherit, filename, lineno = inherits.pop(0)
+        d.setVar('__BBDEFINHERITS', inherits)
+        bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True)
+
     onlyfinalise = d.getVar("__ONLYFINALISE", False)
 
     safe_d = d
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index f8988b8631..c13e4b9755 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -19,11 +19,9 @@ from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
 from .ConfHandler import include, init
 
-# For compatibility
-bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
-
-__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
+__inherit_def_regexp__   = re.compile(r"inherit_defer\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
 __deltask_regexp__       = re.compile(r"deltask\s+(.+)")
@@ -36,6 +34,7 @@ __infunc__ = []
 __inpython__ = False
 __body__   = []
 __classname__ = ""
+__residue__ = []
 
 cached_statements = {}
 
@@ -43,31 +42,46 @@ def supports(fn, d):
     """Return True if fn has a supported extension"""
     return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"]
 
-def inherit(files, fn, lineno, d):
+def inherit(files, fn, lineno, d, deferred=False):
     __inherit_cache = d.getVar('__inherit_cache', False) or []
+    #if "${" in files and not deferred:
+    #    bb.warn("%s:%s has non deferred conditional inherit" % (fn, lineno))
     files = d.expand(files).split()
     for file in files:
-        if not os.path.isabs(file) and not file.endswith(".bbclass"):
-            file = os.path.join('classes', '%s.bbclass' % file)
-
-        if not os.path.isabs(file):
-            bbpath = d.getVar("BBPATH")
-            abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
-            for af in attempts:
-                if af != abs_fn:
-                    bb.parse.mark_dependency(d, af)
-            if abs_fn:
-                file = abs_fn
+        classtype = d.getVar("__bbclasstype", False)
+        origfile = file
+        for t in ["classes-" + classtype, "classes"]:
+            file = origfile
+            if not os.path.isabs(file) and not file.endswith(".bbclass"):
+                file = os.path.join(t, '%s.bbclass' % file)
+
+            if not os.path.isabs(file):
+                bbpath = d.getVar("BBPATH")
+                abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
+                for af in attempts:
+                    if af != abs_fn:
+                        bb.parse.mark_dependency(d, af)
+                if abs_fn:
+                    file = abs_fn
+
+            if os.path.exists(file):
+                break
+
+        if not os.path.exists(file):
+            raise ParseError("Could not inherit file %s" % (file), fn, lineno)
 
         if not file in __inherit_cache:
             logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
-            include(fn, file, lineno, d, "inherit")
+            try:
+                bb.parse.handle(file, d, True)
+            except (IOError, OSError) as exc:
+                raise ParseError("Could not inherit file %s: %s" % (fn, exc.strerror), fn, lineno)
             __inherit_cache = d.getVar('__inherit_cache', False) or []
 
 def get_statements(filename, absolute_filename, base_name):
-    global cached_statements
+    global cached_statements, __residue__, __body__
 
     try:
         return cached_statements[absolute_filename]
@@ -87,12 +101,17 @@ def get_statements(filename, absolute_filename, base_name):
             # add a blank line to close out any python definition
             feeder(lineno, "", filename, base_name, statements, eof=True)
 
+        if __residue__:
+            raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno)
+        if __body__:
+            raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno)
+
         if filename.endswith(".bbclass") or filename.endswith(".inc"):
             cached_statements[absolute_filename] = statements
         return statements
 
-def handle(fn, d, include):
-    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__
+def handle(fn, d, include, baseconfig=False):
+    global __infunc__, __body__, __residue__, __classname__
     __body__ = []
     __infunc__ = []
     __classname__ = ""
@@ -144,7 +163,7 @@ def handle(fn, d, include):
     return d
 
 def feeder(lineno, s, fn, root, statements, eof=False):
-    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__
+    global __inpython__, __infunc__, __body__, __residue__, __classname__
 
     # Check tabs in python functions:
     # - def py_funcname(): covered by __inpython__
@@ -181,10 +200,10 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     if s and s[0] == '#':
         if len(__residue__) != 0 and __residue__[0][0] != "#":
-            bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s))
+            bb.fatal("There is a comment on line %s of file %s:\n'''\n%s\n'''\nwhich is in the middle of a multiline expression. This syntax is invalid, please correct it." % (lineno, fn, s))
 
     if len(__residue__) != 0 and __residue__[0][0] == "#" and (not s or s[0] != "#"):
-        bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+        bb.fatal("There is a confusing multiline partially commented expression on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (lineno - len(__residue__), fn, "\n".join(__residue__)))
 
     if s and s[-1] == '\\':
         __residue__.append(s[:-1])
@@ -255,7 +274,12 @@ def feeder(lineno, s, fn, root, statements, eof=False):
         ast.handleInherit(statements, fn, lineno, m)
         return
 
-    return ConfHandler.feeder(lineno, s, fn, statements)
+    m = __inherit_def_regexp__.match(s)
+    if m:
+        ast.handleInheritDeferred(statements, fn, lineno, m)
+        return
+
+    return ConfHandler.feeder(lineno, s, fn, statements, conffile=False)
 
 # Add us to the handlers list
 from .. import handlers
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index f171c5c932..7826dee7d3 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -20,8 +20,8 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
-    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
+    (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]*)\])?
 
     \s* (
         (?P<colon>:=) |
@@ -45,13 +45,11 @@ __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
-__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]+)\]$" )
+__addpylib_regexp__      = re.compile(r"addpylib\s+(.+)\s+(.+)" )
 
 def init(data):
-    topdir = data.getVar('TOPDIR', False)
-    if not topdir:
-        data.setVar('TOPDIR', os.getcwd())
-
+    return
 
 def supports(fn, d):
     return fn[-5:] == ".conf"
@@ -105,12 +103,12 @@ def include_single_file(parentfn, fn, lineno, data, error_out):
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like assigning
 # a weak default, it turns out to be very difficult to filter out these changes,
-# particularly when the weak default might appear half way though parsing a chain 
+# particularly when the weak default might appear half way though parsing a chain
 # of configuration files. We therefore let the UIs hook into configuration file
 # parsing. This turns out to be a hard problem to solve any other way.
 confFilters = []
 
-def handle(fn, data, include):
+def handle(fn, data, include, baseconfig=False):
     init(data)
 
     if include == 0:
@@ -128,21 +126,26 @@ def handle(fn, data, include):
             s = f.readline()
             if not s:
                 break
+            origlineno = lineno
+            origline = s
             w = s.strip()
             # skip empty lines
             if not w:
                 continue
             s = s.rstrip()
             while s[-1] == '\\':
-                s2 = f.readline().rstrip()
+                line = f.readline()
+                origline += line
+                s2 = line.rstrip()
                 lineno = lineno + 1
                 if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                    bb.fatal("There is a confusing multiline, partially commented expression starting on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (origlineno, fn, origline))
+
                 s = s[:-1] + s2
             # skip comments
             if s[0] == '#':
                 continue
-            feeder(lineno, s, abs_fn, statements)
+            feeder(lineno, s, abs_fn, statements, baseconfig=baseconfig)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)
@@ -150,14 +153,14 @@ def handle(fn, data, include):
     if oldfile:
         data.setVar('FILE', oldfile)
 
-    f.close()
-
     for f in confFilters:
         f(fn, data)
 
     return data
 
-def feeder(lineno, s, fn, statements):
+# baseconfig is set for the bblayers/layer.conf cookerdata config parsing
+# The function is also used by BBHandler, conffile would be False
+def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
@@ -189,6 +192,11 @@ def feeder(lineno, s, fn, statements):
         ast.handleUnsetFlag(statements, fn, lineno, m)
         return
 
+    m = __addpylib_regexp__.match(s)
+    if baseconfig and conffile and m:
+        ast.handlePyLib(statements, fn, lineno, m)
+        return
+
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
index c6a209fb3f..bcca791edf 100644
--- a/bitbake/lib/bb/persist_data.py
+++ b/bitbake/lib/bb/persist_data.py
@@ -12,14 +12,14 @@ currently, providing a key/value store accessed by 'domain'.
 #
 
 import collections
+import collections.abc
 import contextlib
 import functools
 import logging
 import os.path
 import sqlite3
 import sys
-import warnings
-from collections import Mapping
+from collections.abc import Mapping
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -29,7 +29,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 logger = logging.getLogger("BitBake.PersistData")
 
 @functools.total_ordering
-class SQLTable(collections.MutableMapping):
+class SQLTable(collections.abc.MutableMapping):
     class _Decorators(object):
         @staticmethod
         def retry(*, reconnect=True):
@@ -63,7 +63,7 @@ class SQLTable(collections.MutableMapping):
             """
             Decorator that starts a database transaction and creates a database
             cursor for performing queries. If no exception is thrown, the
-            database results are commited. If an exception occurs, the database
+            database results are committed. If an exception occurs, the database
             is rolled back. In all cases, the cursor is closed after the
             function ends.
 
@@ -208,7 +208,7 @@ class SQLTable(collections.MutableMapping):
 
     def __lt__(self, other):
         if not isinstance(other, Mapping):
-            raise NotImplemented
+            raise NotImplementedError()
 
         return len(self) < len(other)
 
@@ -238,55 +238,6 @@ class SQLTable(collections.MutableMapping):
     def has_key(self, key):
         return key in self
 
-
-class PersistData(object):
-    """Deprecated representation of the bitbake persistent data store"""
-    def __init__(self, d):
-        warnings.warn("Use of PersistData is deprecated.  Please use "
-                      "persist(domain, d) instead.",
-                      category=DeprecationWarning,
-                      stacklevel=2)
-
-        self.data = persist(d)
-        logger.debug("Using '%s' as the persistent data cache",
-                     self.data.filename)
-
-    def addDomain(self, domain):
-        """
-        Add a domain (pending deprecation)
-        """
-        return self.data[domain]
-
-    def delDomain(self, domain):
-        """
-        Removes a domain and all the data it contains
-        """
-        del self.data[domain]
-
-    def getKeyValues(self, domain):
-        """
-        Return a list of key + value pairs for a domain
-        """
-        return list(self.data[domain].items())
-
-    def getValue(self, domain, key):
-        """
-        Return the value of a key for a domain
-        """
-        return self.data[domain][key]
-
-    def setValue(self, domain, key, value):
-        """
-        Sets the value of a key for a domain
-        """
-        self.data[domain][key] = value
-
-    def delValue(self, domain, key):
-        """
-        Deletes a key/value pair
-        """
-        del self.data[domain][key]
-
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
     import bb.utils
@@ -298,4 +249,23 @@ def persist(domain, d):
 
     bb.utils.mkdirhier(cachedir)
     cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
-    return SQLTable(cachefile, domain)
+
+    try:
+        return SQLTable(cachefile, domain)
+    except sqlite3.OperationalError:
+        # Sqlite fails to open database when its path is too long.
+        # After testing, 504 is the biggest path length that can be opened by
+        # sqlite.
+        # Note: This code is called before sanity.bbclass and its path length
+        # check
+        max_len = 504
+        if len(cachefile) > max_len:
+            logger.critical("The path of the cache file is too long "
+                    "({0} chars > {1}) to be opened by sqlite! "
+                    "Your cache file is \"{2}\"".format(
+                        len(cachefile),
+                        max_len,
+                        cachefile))
+            sys.exit(1)
+        else:
+            raise
diff --git a/bitbake/lib/bb/process.py b/bitbake/lib/bb/process.py
index 7c3995cce5..4c7b6d39df 100644
--- a/bitbake/lib/bb/process.py
+++ b/bitbake/lib/bb/process.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -60,7 +62,7 @@ class Popen(subprocess.Popen):
         "close_fds": True,
         "preexec_fn": subprocess_setup,
         "stdout": subprocess.PIPE,
-        "stderr": subprocess.STDOUT,
+        "stderr": subprocess.PIPE,
         "stdin": subprocess.PIPE,
         "shell": False,
     }
@@ -142,7 +144,7 @@ def _logged_communicate(pipe, log, input, extrafiles):
         while pipe.poll() is None:
             read_all_pipes(log, rin, outdata, errdata)
 
-        # Pocess closed, drain all pipes...
+        # Process closed, drain all pipes...
         read_all_pipes(log, rin, outdata, errdata)
     finally:
         log.flush()
@@ -181,5 +183,8 @@ def run(cmd, input=None, log=None, extrafiles=None, **options):
             stderr = stderr.decode("utf-8")
 
     if pipe.returncode != 0:
+        if log:
+            # Don't duplicate the output in the exception if logging it
+            raise ExecutionError(cmd, pipe.returncode, None, None)
         raise ExecutionError(cmd, pipe.returncode, stdout, stderr)
     return stdout, stderr
diff --git a/bitbake/lib/bb/progress.py b/bitbake/lib/bb/progress.py
index d051ba0198..9518be77fb 100644
--- a/bitbake/lib/bb/progress.py
+++ b/bitbake/lib/bb/progress.py
@@ -94,12 +94,15 @@ class LineFilterProgressHandler(ProgressHandler):
         while True:
             breakpos = self._linebuffer.find('\n') + 1
             if breakpos == 0:
-                break
+                # for the case when the line with progress ends with only '\r'
+                breakpos = self._linebuffer.find('\r') + 1
+                if breakpos == 0:
+                    break
             line = self._linebuffer[:breakpos]
             self._linebuffer = self._linebuffer[breakpos:]
             # Drop any line feeds and anything that precedes them
             lbreakpos = line.rfind('\r') + 1
-            if lbreakpos:
+            if lbreakpos and lbreakpos != breakpos:
                 line = line[lbreakpos:]
             if self.writeline(filter_color(line)):
                 super().write(line)
@@ -145,7 +148,7 @@ class MultiStageProgressReporter:
     for tasks made up of python code spread across multiple
     classes / functions - the progress reporter object can
     be passed around or stored at the object level and calls
-    to next_stage() and update() made whereever needed.
+    to next_stage() and update() made wherever needed.
     """
     def __init__(self, d, stage_weights, debug=False):
         """
diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py
index b5a6cd0090..e11a4637d1 100644
--- a/bitbake/lib/bb/providers.py
+++ b/bitbake/lib/bb/providers.py
@@ -38,16 +38,17 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
     localdata = data.createCopy(cfgData)
     bb.data.expandKeys(localdata)
 
+    required = {}
     preferred_versions = {}
     latest_versions = {}
 
     for pn in pkg_pn:
-        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
         preferred_versions[pn] = (pref_ver, pref_file)
         latest_versions[pn] = (last_ver, last_file)
+        required[pn] = req
 
-    return (latest_versions, preferred_versions)
-
+    return (latest_versions, preferred_versions, required)
 
 def allProviders(dataCache):
     """
@@ -59,7 +60,6 @@ def allProviders(dataCache):
         all_providers[pn].append((ver, fn))
     return all_providers
 
-
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference
@@ -87,6 +87,21 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
 
     return tmp_pn
 
+def versionVariableMatch(cfgData, keyword, pn):
+    """
+    Return the value of the <keyword>_VERSION variable if set.
+    """
+
+    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
+    # hence we do this manually rather than use OVERRIDES
+    ver = cfgData.getVar("%s_VERSION:pn-%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION" % keyword)
+
+    return ver
+
 def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
     """
     Check if the version pe,pv,pr is the preferred one.
@@ -102,19 +117,28 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
 
 def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
-    Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+    Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
     """
 
     preferred_file = None
     preferred_ver = None
+    required = False
 
-    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
-    # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION")
+    required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
+    preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
+
+    itemstr = ""
+    if item:
+        itemstr = " (for item %s)" % item
+
+    if required_v is not None:
+        if preferred_v is not None:
+            logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
+        else:
+            logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
+        # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
+        preferred_v = required_v
+        required = True
 
     if preferred_v:
         m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
@@ -147,11 +171,9 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             pv_str = preferred_v
         if not (preferred_e is None):
             pv_str = '%s:%s' % (preferred_e, pv_str)
-        itemstr = ""
-        if item:
-            itemstr = " (for item %s)" % item
         if preferred_file is None:
-            logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+            if not required:
+                logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
             available_vers = []
             for file_set in pkg_pn:
                 for f in file_set:
@@ -163,12 +185,16 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
                         available_vers.append(ver_str)
             if available_vers:
                 available_vers.sort()
-                logger.warn("versions of %s available: %s", pn, ' '.join(available_vers))
+                logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
+            if required:
+                logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
         else:
-            logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
-
-    return (preferred_ver, preferred_file)
+            if required:
+                logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+            else:
+                logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
 
+    return (preferred_ver, preferred_file, required)
 
 def findLatestProvider(pn, cfgData, dataCache, file_set):
     """
@@ -189,7 +215,6 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
 
     return (latest, latest_f)
 
-
 def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
     If there is a PREFERRED_VERSION, find the highest-priority bbfile
@@ -198,17 +223,16 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
 
     sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
-    # Find the highest priority provider with a PREFERRED_VERSION set
-    (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+    # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
+    (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
     # Find the latest version of the highest priority provider
     (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
 
-    if preferred_file is None:
+    if not required and preferred_file is None:
         preferred_file = latest_f
         preferred_ver = latest
 
-    return (latest, latest_f, preferred_ver, preferred_file)
-
+    return (latest, latest_f, preferred_ver, preferred_file, required)
 
 def _filterProviders(providers, item, cfgData, dataCache):
     """
@@ -234,10 +258,13 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
 
-    # First add PREFERRED_VERSIONS
+    # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
     for pn in sorted(pkg_pn):
         sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
-        preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        if required and preferred_file is None:
+            return eligible
+        preferred_versions[pn] = (preferred_ver, preferred_file)
         if preferred_versions[pn][1]:
             eligible.append(preferred_versions[pn][1])
 
@@ -249,7 +276,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
         eligible.append(preferred_versions[pn][1])
 
     if not eligible:
-        logger.error("no eligible providers for %s", item)
         return eligible
 
     # If pn == item, give it a slight default preference
@@ -266,7 +292,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     return eligible
 
-
 def filterProviders(providers, item, cfgData, dataCache):
     """
     Take a list of providers and filter/reorder according to the
@@ -371,8 +396,8 @@ def getRuntimeProviders(dataCache, rdepend):
         return rproviders
 
     # Only search dynamic packages if we can't find anything in other variables
-    for pattern in dataCache.packages_dynamic:
-        pattern = pattern.replace(r'+', r"\+")
+    for pat_key in dataCache.packages_dynamic:
+        pattern = pat_key.replace(r'+', r"\+")
         if pattern in regexp_cache:
             regexp = regexp_cache[pattern]
         else:
@@ -383,12 +408,11 @@ def getRuntimeProviders(dataCache, rdepend):
                 raise
             regexp_cache[pattern] = regexp
         if regexp.match(rdepend):
-            rproviders += dataCache.packages_dynamic[pattern]
+            rproviders += dataCache.packages_dynamic[pat_key]
             logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
 
     return rproviders
 
-
 def buildWorldTargetList(dataCache, task=None):
     """
     Build package list for "bitbake world"
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 54ef245a63..bc7e18175d 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -24,6 +24,7 @@ import pickle
 from multiprocessing import Process
 import shlex
 import pprint
+import time
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
@@ -85,15 +86,19 @@ class RunQueueStats:
     """
     Holds statistics on the tasks handled by the associated runQueue
     """
-    def __init__(self, total):
+    def __init__(self, total, setscene_total):
         self.completed = 0
         self.skipped = 0
         self.failed = 0
         self.active = 0
+        self.setscene_active = 0
+        self.setscene_covered = 0
+        self.setscene_notcovered = 0
+        self.setscene_total = setscene_total
         self.total = total
 
     def copy(self):
-        obj = self.__class__(self.total)
+        obj = self.__class__(self.total, self.setscene_total)
         obj.__dict__.update(self.__dict__)
         return obj
 
@@ -112,6 +117,13 @@ class RunQueueStats:
     def taskActive(self):
         self.active = self.active + 1
 
+    def updateCovered(self, covered, notcovered):
+        self.setscene_covered = covered
+        self.setscene_notcovered = notcovered
+
+    def updateActiveSetscene(self, active):
+        self.setscene_active = active
+
 # These values indicate the next step due to be run in the
 # runQueue state machine
 runQueuePrepare = 2
@@ -143,11 +155,82 @@ class RunQueueScheduler(object):
         self.stamps = {}
         for tid in self.rqdata.runtaskentries:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-            self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             if tid in self.rq.runq_buildable:
-                self.buildable.append(tid)
+                self.buildable.add(tid)
 
         self.rev_prio_map = None
+        self.is_pressure_usable()
+
+    def is_pressure_usable(self):
+        """
+        If monitoring pressure, return True if pressure files can be open and read. For example
+        openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported)
+        is returned.
+        """
+        if self.rq.max_cpu_pressure or self.rq.max_io_pressure or self.rq.max_memory_pressure:
+            try:
+                with open("/proc/pressure/cpu") as cpu_pressure_fds, \
+                    open("/proc/pressure/io") as io_pressure_fds, \
+                    open("/proc/pressure/memory") as memory_pressure_fds:
+
+                    self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_pressure_time = time.time()
+                self.check_pressure = True
+            except:
+                bb.note("The /proc/pressure files can't be read. Continuing build without monitoring pressure")
+                self.check_pressure = False
+        else:
+            self.check_pressure = False
+
+    def exceeds_max_pressure(self):
+        """
+        Monitor the difference in total pressure at least once per second, if
+        BB_PRESSURE_MAX_{CPU|IO|MEMORY} are set, return True if above threshold.
+        """
+        if self.check_pressure:
+            with open("/proc/pressure/cpu") as cpu_pressure_fds, \
+                open("/proc/pressure/io") as io_pressure_fds, \
+                open("/proc/pressure/memory") as memory_pressure_fds:
+                # extract "total" from /proc/pressure/{cpu|io}
+                curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
+                curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
+                curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
+                now = time.time()
+                tdiff = now - self.prev_pressure_time
+                psi_accumulation_interval = 1.0
+                cpu_pressure = (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) / tdiff
+                io_pressure = (float(curr_io_pressure) - float(self.prev_io_pressure)) / tdiff
+                memory_pressure = (float(curr_memory_pressure) - float(self.prev_memory_pressure)) / tdiff
+                exceeds_cpu_pressure =  self.rq.max_cpu_pressure and cpu_pressure > self.rq.max_cpu_pressure
+                exceeds_io_pressure =  self.rq.max_io_pressure and io_pressure > self.rq.max_io_pressure
+                exceeds_memory_pressure =  self.rq.max_memory_pressure and memory_pressure > self.rq.max_memory_pressure
+
+                if tdiff > psi_accumulation_interval:
+                    self.prev_cpu_pressure = curr_cpu_pressure
+                    self.prev_io_pressure = curr_io_pressure
+                    self.prev_memory_pressure = curr_memory_pressure
+                    self.prev_pressure_time = now
+
+            pressure_state = (exceeds_cpu_pressure, exceeds_io_pressure, exceeds_memory_pressure)
+            pressure_values = (round(cpu_pressure,1), self.rq.max_cpu_pressure, round(io_pressure,1), self.rq.max_io_pressure, round(memory_pressure,1), self.rq.max_memory_pressure)
+            if hasattr(self, "pressure_state") and pressure_state != self.pressure_state:
+                bb.note("Pressure status changed to CPU: %s, IO: %s, Mem: %s (CPU: %s/%s, IO: %s/%s, Mem: %s/%s) - using %s/%s bitbake threads" % (pressure_state + pressure_values + (len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks)))
+            self.pressure_state = pressure_state
+            return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure)
+        elif self.rq.max_loadfactor:
+            limit = False
+            loadfactor = float(os.getloadavg()[0]) / os.cpu_count()
+            # bb.warn("Comparing %s to %s" % (loadfactor, self.rq.max_loadfactor))
+            if loadfactor > self.rq.max_loadfactor:
+                limit = True
+            if hasattr(self, "loadfactor_limit") and limit != self.loadfactor_limit:
+                bb.note("Load average limiting set to %s as load average: %s - using %s/%s bitbake threads" % (limit, loadfactor, len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks))
+            self.loadfactor_limit = limit
+            return limit
+        return False
 
     def next_buildable_task(self):
         """
@@ -161,6 +244,12 @@ class RunQueueScheduler(object):
         if not buildable:
             return None
 
+        # Bitbake requires that at least one task be active. Only check for pressure if
+        # this is the case, otherwise the pressure limitation could result in no tasks
+        # being active and no new tasks started thereby, at times, breaking the scheduler.
+        if self.rq.stats.active and self.exceeds_max_pressure():
+            return None
+
         # Filter out tasks that have a max number of threads that have been exceeded
         skip_buildable = {}
         for running in self.rq.runq_running.difference(self.rq.runq_complete):
@@ -191,11 +280,11 @@ class RunQueueScheduler(object):
         best = None
         bestprio = None
         for tid in buildable:
-            taskname = taskname_from_tid(tid)
-            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
-                continue
             prio = self.rev_prio_map[tid]
             if bestprio is None or bestprio > prio:
+                taskname = taskname_from_tid(tid)
+                if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                    continue
                 stamp = self.stamps[tid]
                 if stamp in self.rq.build_stamps.values():
                     continue
@@ -374,10 +463,9 @@ class RunQueueData:
         self.rq = rq
         self.warn_multi_bb = False
 
-        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or ""
-        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
-        self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData, targets)
-        self.setscenewhitelist_checked = False
+        self.multi_provider_allowed = (cfgData.getVar("BB_MULTI_PROVIDER_ALLOWED") or "").split()
+        self.setscene_ignore_tasks = get_setscene_enforce_ignore_tasks(cfgData, targets)
+        self.setscene_ignore_tasks_checked = False
         self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1")
         self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
@@ -475,7 +563,7 @@ class RunQueueData:
                             msgs.append("  Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends)))
                         msgs.append("\n")
                     if len(valid_chains) > 10:
-                        msgs.append("Aborted dependency loops search after 10 matches.\n")
+                        msgs.append("Halted dependency loops search after 10 matches.\n")
                         raise TooManyLoops
                     continue
                 scan = False
@@ -536,7 +624,7 @@ class RunQueueData:
                         next_points.append(revdep)
                         task_done[revdep] = True
             endpoints = next_points
-            if len(next_points) == 0:
+            if not next_points:
                 break
 
         # Circular dependency sanity check
@@ -578,15 +666,18 @@ class RunQueueData:
 
         found = False
         for mc in self.taskData:
-            if len(taskData[mc].taskentries) > 0:
+            if taskData[mc].taskentries:
                 found = True
                 break
         if not found:
             # Nothing to do
             return 0
 
+        bb.parse.siggen.setup_datacache(self.dataCaches)
+
         self.init_progress_reporter.start()
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Step A - Work out a list of tasks to run
         #
@@ -632,6 +723,8 @@ class RunQueueData:
                 frommc = mcdependency[1]
                 mcdep = mcdependency[2]
                 deptask = mcdependency[4]
+                if mcdep not in taskData:
+                    bb.fatal("Multiconfig '%s' is referenced in multiconfig dependency '%s' but not enabled in BBMULTICONFIG?" % (mcdep, dep))
                 if mc == frommc:
                     fn = taskData[mcdep].build_targets[pn][0]
                     newdep = '%s:%s' % (fn,deptask)
@@ -733,6 +826,7 @@ class RunQueueData:
         #self.dump_data()
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Resolve recursive 'recrdeptask' dependencies (Part B)
         #
@@ -762,7 +856,7 @@ class RunQueueData:
         # Find the dependency chain endpoints
         endpoints = set()
         for tid in self.runtaskentries:
-            if len(deps[tid]) == 0:
+            if not deps[tid]:
                 endpoints.add(tid)
         # Iterate the chains collating dependencies
         while endpoints:
@@ -773,11 +867,11 @@ class RunQueueData:
                     cumulativedeps[dep].update(cumulativedeps[tid])
                     if tid in deps[dep]:
                         deps[dep].remove(tid)
-                    if len(deps[dep]) == 0:
+                    if not deps[dep]:
                         next.add(dep)
             endpoints = next
         #for tid in deps:
-        #    if len(deps[tid]) != 0:
+        #    if deps[tid]:
         #        bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid]))
 
         # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to
@@ -829,6 +923,7 @@ class RunQueueData:
             self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         #self.dump_data()
 
@@ -867,7 +962,7 @@ class RunQueueData:
                     bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname)
             else:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
-                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], taskfn)
+                bb.parse.siggen.invalidate_task(taskname, taskfn)
 
         self.target_tids = []
         for (mc, target, task, fn) in self.targets:
@@ -910,47 +1005,54 @@ class RunQueueData:
                 mark_active(tid, 1)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Step C - Prune all inactive tasks
         #
         # Once all active tasks are marked, prune the ones we don't need.
 
-        delcount = {}
-        for tid in list(self.runtaskentries.keys()):
-            if tid not in runq_build:
-                delcount[tid] = self.runtaskentries[tid]
-                del self.runtaskentries[tid]
-
         # Handle --runall
         if self.cooker.configuration.runall:
             # re-run the mark_active and then drop unused tasks from new list
-            runq_build = {}
 
-            for task in self.cooker.configuration.runall:
-                if not task.startswith("do_"):
-                    task = "do_{0}".format(task)
-                runall_tids = set()
-                for tid in list(self.runtaskentries):
-                    wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
-                    if wanttid in delcount:
-                        self.runtaskentries[wanttid] = delcount[wanttid]
-                    if wanttid in self.runtaskentries:
-                        runall_tids.add(wanttid)
-
-                for tid in list(runall_tids):
-                    mark_active(tid,1)
-                    if self.cooker.configuration.force:
-                        invalidate_task(tid, False)
+            runall_tids = set()
+            added = True
+            while added:
+                reduced_tasklist = set(self.runtaskentries.keys())
+                for tid in list(self.runtaskentries.keys()):
+                    if tid not in runq_build:
+                       reduced_tasklist.remove(tid)
+                runq_build = {}
 
-            for tid in list(self.runtaskentries.keys()):
-                if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
-                    del self.runtaskentries[tid]
+                orig = runall_tids
+                runall_tids = set()
+                for task in self.cooker.configuration.runall:
+                    if not task.startswith("do_"):
+                        task = "do_{0}".format(task)
+                    for tid in reduced_tasklist:
+                        wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
+                        if wanttid in self.runtaskentries:
+                            runall_tids.add(wanttid)
+
+                    for tid in list(runall_tids):
+                        mark_active(tid, 1)
+                        self.target_tids.append(tid)
+                        if self.cooker.configuration.force:
+                            invalidate_task(tid, False)
+                added = runall_tids - orig
+
+        delcount = set()
+        for tid in list(self.runtaskentries.keys()):
+            if tid not in runq_build:
+                delcount.add(tid)
+                del self.runtaskentries[tid]
 
-            if len(self.runtaskentries) == 0:
+        if self.cooker.configuration.runall:
+            if not self.runtaskentries:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Handle runonly
         if self.cooker.configuration.runonly:
@@ -960,19 +1062,19 @@ class RunQueueData:
             for task in self.cooker.configuration.runonly:
                 if not task.startswith("do_"):
                     task = "do_{0}".format(task)
-                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
+                runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task]
 
-                for tid in list(runonly_tids):
-                    mark_active(tid,1)
+                for tid in runonly_tids:
+                    mark_active(tid, 1)
                     if self.cooker.configuration.force:
                         invalidate_task(tid, False)
 
             for tid in list(self.runtaskentries.keys()):
                 if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
+                    delcount.add(tid)
                     del self.runtaskentries[tid]
 
-            if len(self.runtaskentries) == 0:
+            if not self.runtaskentries:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets)))
 
         #
@@ -980,8 +1082,8 @@ class RunQueueData:
         #
 
         # Check to make sure we still have tasks to run
-        if len(self.runtaskentries) == 0:
-            if not taskData[''].abort:
+        if not self.runtaskentries:
+            if not taskData[''].halt:
                 bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.")
             else:
                 bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.")
@@ -991,6 +1093,7 @@ class RunQueueData:
         logger.verbose("Assign Weightings")
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Generate a list of reverse dependencies to ease future calculations
         for tid in self.runtaskentries:
@@ -998,13 +1101,14 @@ class RunQueueData:
                 self.runtaskentries[dep].revdeps.add(tid)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Identify tasks at the end of dependency chains
         # Error on circular dependency loops (length two)
         endpoints = []
         for tid in self.runtaskentries:
             revdeps = self.runtaskentries[tid].revdeps
-            if len(revdeps) == 0:
+            if not revdeps:
                 endpoints.append(tid)
             for dep in revdeps:
                 if dep in self.runtaskentries[tid].depends:
@@ -1014,12 +1118,14 @@ class RunQueueData:
         logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Calculate task weights
         # Check of higher length circular dependencies
         self.runq_weight = self.calculate_task_weights(endpoints)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Sanity Check - Check for multiple tasks building the same provider
         for mc in self.dataCaches:
@@ -1040,7 +1146,7 @@ class RunQueueData:
             for prov in prov_list:
                 if len(prov_list[prov]) < 2:
                     continue
-                if prov in self.multi_provider_whitelist:
+                if prov in self.multi_provider_allowed:
                     continue
                 seen_pn = []
                 # If two versions of the same PN are being built its fatal, we don't support it.
@@ -1050,12 +1156,12 @@ class RunQueueData:
                         seen_pn.append(pn)
                     else:
                         bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn))
-                msg = "Multiple .bb files are due to be built which each provide %s:\n  %s" % (prov, "\n  ".join(prov_list[prov]))
+                msgs = ["Multiple .bb files are due to be built which each provide %s:\n  %s" % (prov, "\n  ".join(prov_list[prov]))]
                 #
                 # Construct a list of things which uniquely depend on each provider
                 # since this may help the user figure out which dependency is triggering this warning
                 #
-                msg += "\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from."
+                msgs.append("\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from.")
                 deplist = {}
                 commondeps = None
                 for provfn in prov_list[prov]:
@@ -1075,12 +1181,12 @@ class RunQueueData:
                         commondeps &= deps
                     deplist[provfn] = deps
                 for provfn in deplist:
-                    msg += "\n%s has unique dependees:\n  %s" % (provfn, "\n  ".join(deplist[provfn] - commondeps))
+                    msgs.append("\n%s has unique dependees:\n  %s" % (provfn, "\n  ".join(deplist[provfn] - commondeps)))
                 #
                 # Construct a list of provides and runtime providers for each recipe
                 # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC)
                 #
-                msg += "\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful."
+                msgs.append("\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful.")
                 provide_results = {}
                 rprovide_results = {}
                 commonprovs = None
@@ -1107,30 +1213,20 @@ class RunQueueData:
                     else:
                         commonrprovs &= rprovides
                     rprovide_results[provfn] = rprovides
-                #msg += "\nCommon provides:\n  %s" % ("\n  ".join(commonprovs))
-                #msg += "\nCommon rprovides:\n  %s" % ("\n  ".join(commonrprovs))
+                #msgs.append("\nCommon provides:\n  %s" % ("\n  ".join(commonprovs)))
+                #msgs.append("\nCommon rprovides:\n  %s" % ("\n  ".join(commonrprovs)))
                 for provfn in prov_list[prov]:
-                    msg += "\n%s has unique provides:\n  %s" % (provfn, "\n  ".join(provide_results[provfn] - commonprovs))
-                    msg += "\n%s has unique rprovides:\n  %s" % (provfn, "\n  ".join(rprovide_results[provfn] - commonrprovs))
+                    msgs.append("\n%s has unique provides:\n  %s" % (provfn, "\n  ".join(provide_results[provfn] - commonprovs)))
+                    msgs.append("\n%s has unique rprovides:\n  %s" % (provfn, "\n  ".join(rprovide_results[provfn] - commonrprovs)))
 
                 if self.warn_multi_bb:
-                    logger.verbnote(msg)
+                    logger.verbnote("".join(msgs))
                 else:
-                    logger.error(msg)
+                    logger.error("".join(msgs))
 
         self.init_progress_reporter.next_stage()
-
-        # Create a whitelist usable by the stamp checks
-        self.stampfnwhitelist = {}
-        for mc in self.taskData:
-            self.stampfnwhitelist[mc] = []
-            for entry in self.stampwhitelist.split():
-                if entry not in self.taskData[mc].build_targets:
-                    continue
-                fn = self.taskData.build_targets[entry][0]
-                self.stampfnwhitelist[mc].append(fn)
-
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Iterate over the task list looking for tasks with a 'setscene' function
         self.runq_setscene_tids = set()
@@ -1143,6 +1239,7 @@ class RunQueueData:
                 self.runq_setscene_tids.add(tid)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Invalidate task if force mode active
         if self.cooker.configuration.force:
@@ -1159,6 +1256,7 @@ class RunQueueData:
                     invalidate_task(fn + ":" + st, True)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Create and print to the logs a virtual/xxxx -> PN (fn) table
         for mc in taskData:
@@ -1171,18 +1269,20 @@ class RunQueueData:
                 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCaches[mc])
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
-        while len(todeal) > 0:
+        while todeal:
             for tid in todeal.copy():
-                if len(self.runtaskentries[tid].depends - dealtwith) == 0:
+                if not (self.runtaskentries[tid].depends - dealtwith):
                     dealtwith.add(tid)
                     todeal.remove(tid)
                     self.prepare_task_hash(tid)
+                bb.event.check_for_interrupts(self.cooker.data)
 
         bb.parse.siggen.writeout_file_checksum_cache()
 
@@ -1190,9 +1290,8 @@ class RunQueueData:
         return len(self.runtaskentries)
 
     def prepare_task_hash(self, tid):
-        dc = bb.parse.siggen.get_data_caches(self.dataCaches, mc_from_tid(tid))
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, dc)
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, dc)
+        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
         self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
 
     def dump_data(self):
@@ -1218,7 +1317,6 @@ class RunQueue:
         self.cfgData = cfgData
         self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets)
 
-        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile"
         self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
         self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
 
@@ -1237,30 +1335,40 @@ class RunQueue:
         self.worker = {}
         self.fakeworker = {}
 
+    @staticmethod
+    def send_pickled_data(worker, data, name):
+        msg = bytearray()
+        msg.extend(b"<" + name.encode() + b">")
+        pickled_data = pickle.dumps(data)
+        msg.extend(len(pickled_data).to_bytes(4, 'big'))
+        msg.extend(pickled_data)
+        msg.extend(b"</" + name.encode() + b">")
+        worker.stdin.write(msg)
+
     def _start_worker(self, mc, fakeroot = False, rqexec = None):
         logger.debug("Starting bitbake-worker")
         magic = "decafbad"
         if self.cooker.configuration.profile:
             magic = "decafbadbad"
+        fakerootlogs = None
+
+        workerscript = os.path.realpath(os.path.dirname(__file__) + "/../../bin/bitbake-worker")
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
             fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD"))
             fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
-            for key, value in (var.split('=') for var in fakerootenv):
+            for key, value in (var.split('=',1) for var in fakerootenv):
                 env[key] = value
-            worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            worker = subprocess.Popen(fakerootcmd + [sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
         else:
-            worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            worker = subprocess.Popen([sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
-        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
 
         workerdata = {
-            "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
-            "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv,
-            "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
-            "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
             "sigdata" : bb.parse.siggen.get_taskdata(),
             "logdefaultlevel" : bb.msg.loggerDefaultLogLevel,
             "build_verbose_shell" : self.cooker.configuration.build_verbose_shell,
@@ -1274,9 +1382,9 @@ class RunQueue:
             "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"),
         }
 
-        worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
-        worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
-        worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
+        RunQueue.send_pickled_data(worker, self.cooker.configuration, "cookerconfig")
+        RunQueue.send_pickled_data(worker, self.cooker.extraconfigdata, "extraconfigdata")
+        RunQueue.send_pickled_data(worker, workerdata, "workerdata")
         worker.stdin.flush()
 
         return RunQueueWorker(worker, workerpipe)
@@ -1286,7 +1394,7 @@ class RunQueue:
             return
         logger.debug("Teardown for bitbake-worker")
         try:
-           worker.process.stdin.write(b"<quit></quit>")
+           RunQueue.send_pickled_data(worker.process, b"", "quit")
            worker.process.stdin.flush()
            worker.process.stdin.close()
         except IOError:
@@ -1298,12 +1406,12 @@ class RunQueue:
             continue
         worker.pipe.close()
 
-    def start_worker(self):
+    def start_worker(self, rqexec):
         if self.worker:
             self.teardown_workers()
         self.teardown = False
         for mc in self.rqdata.dataCaches:
-            self.worker[mc] = self._start_worker(mc)
+            self.worker[mc] = self._start_worker(mc, False, rqexec)
 
     def start_fakeworker(self, rqexec, mc):
         if not mc in self.fakeworker:
@@ -1345,15 +1453,7 @@ class RunQueue:
         if taskname is None:
             taskname = tn
 
-        if self.stamppolicy == "perfile":
-            fulldeptree = False
-        else:
-            fulldeptree = True
-            stampwhitelist = []
-            if self.stamppolicy == "whitelist":
-                stampwhitelist = self.rqdata.stampfnwhitelist[mc]
-
-        stampfile = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn)
+        stampfile = bb.parse.siggen.stampfile_mcfn(taskname, taskfn)
 
         # If the stamp is missing, it's not current
         if not os.access(stampfile, os.F_OK):
@@ -1365,7 +1465,7 @@ class RunQueue:
             logger.debug2("%s.%s is nostamp\n", fn, taskname)
             return False
 
-        if taskname != "do_setscene" and taskname.endswith("_setscene"):
+        if taskname.endswith("_setscene"):
             return True
 
         if cache is None:
@@ -1376,15 +1476,15 @@ class RunQueue:
         for dep in self.rqdata.runtaskentries[tid].depends:
             if iscurrent:
                 (mc2, fn2, taskname2, taskfn2) = split_tid_mcfn(dep)
-                stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCaches[mc2], taskfn2)
-                stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCaches[mc2], taskfn2)
+                stampfile2 = bb.parse.siggen.stampfile_mcfn(taskname2, taskfn2)
+                stampfile3 = bb.parse.siggen.stampfile_mcfn(taskname2 + "_setscene", taskfn2)
                 t2 = get_timestamp(stampfile2)
                 t3 = get_timestamp(stampfile3)
                 if t3 and not t2:
                     continue
                 if t3 and t3 > t2:
                     continue
-                if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
+                if fn == fn2:
                     if not t2:
                         logger.debug2('Stampfile %s does not exist', stampfile2)
                         iscurrent = False
@@ -1434,10 +1534,11 @@ class RunQueue:
         """
         Run the tasks in a queue prepared by rqdata.prepare()
         Upon failure, optionally try to recover the build using any alternate providers
-        (if the abort on failure configuration option isn't set)
+        (if the halt on failure configuration option isn't set)
         """
 
         retval = True
+        bb.event.check_for_interrupts(self.cooker.data)
 
         if self.state is runQueuePrepare:
             # NOTE: if you add, remove or significantly refactor the stages of this
@@ -1466,10 +1567,13 @@ class RunQueue:
 
             if not self.dm_event_handler_registered:
                  res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
+                                         lambda x, y: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
                                          ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
+            self.rqdata.init_progress_reporter.next_stage()
+            self.rqexe = RunQueueExecute(self)
+
             dump = self.cooker.configuration.dump_signatures
             if dump:
                 self.rqdata.init_progress_reporter.finish()
@@ -1481,16 +1585,14 @@ class RunQueue:
                 self.state = runQueueComplete
 
         if self.state is runQueueSceneInit:
-            self.rqdata.init_progress_reporter.next_stage()
-            self.start_worker()
-            self.rqdata.init_progress_reporter.next_stage()
-            self.rqexe = RunQueueExecute(self)
+            self.start_worker(self.rqexe)
+            self.rqdata.init_progress_reporter.finish()
 
             # If we don't have any setscene functions, skip execution
-            if len(self.rqdata.runq_setscene_tids) == 0:
+            if not self.rqdata.runq_setscene_tids:
                 logger.info('No setscene tasks')
                 for tid in self.rqdata.runtaskentries:
-                    if len(self.rqdata.runtaskentries[tid].depends) == 0:
+                    if not self.rqdata.runtaskentries[tid].depends:
                         self.rqexe.setbuildable(tid)
                     self.rqexe.tasks_notcovered.add(tid)
                 self.rqexe.sqdone = True
@@ -1563,29 +1665,28 @@ class RunQueue:
         else:
             self.rqexe.finish()
 
-    def rq_dump_sigfn(self, fn, options):
-        bb_cache = bb.cache.NoCache(self.cooker.databuilder)
-        mc = bb.runqueue.mc_from_tid(fn)
-        the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn))
-        siggen = bb.parse.siggen
-        dataCaches = self.rqdata.dataCaches
-        siggen.dump_sigfn(fn, dataCaches, options)
+    def _rq_dump_sigtid(self, tids):
+        for tid in tids:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            dataCaches = self.rqdata.dataCaches
+            bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True)
 
     def dump_signatures(self, options):
-        fns = set()
-        bb.note("Reparsing files to collect dependency data")
+        if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
+            bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
 
-        for tid in self.rqdata.runtaskentries:
-            fn = fn_from_tid(tid)
-            fns.add(fn)
+        bb.note("Writing task signature files")
 
         max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+        def chunkify(l, n):
+            return [l[i::n] for i in range(n)]
+        tids = chunkify(list(self.rqdata.runtaskentries), max_process)
         # We cannot use the real multiprocessing.Pool easily due to some local data
         # that can't be pickled. This is a cheap multi-process solution.
         launched = []
-        while fns:
+        while tids:
             if len(launched) < max_process:
-                p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
+                p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), ))
                 p.start()
                 launched.append(p)
             for q in launched:
@@ -1600,6 +1701,17 @@ class RunQueue:
         return
 
     def print_diffscenetasks(self):
+        def get_root_invalid_tasks(task, taskdepends, valid, noexec, visited_invalid):
+            invalidtasks = []
+            for t in taskdepends[task].depends:
+                if t not in valid and t not in visited_invalid:
+                    invalidtasks.extend(get_root_invalid_tasks(t, taskdepends, valid, noexec, visited_invalid))
+                    visited_invalid.add(t)
+
+            direct_invalid = [t for t in taskdepends[task].depends if t not in valid]
+            if not direct_invalid and task not in noexec:
+                invalidtasks = [task]
+            return invalidtasks
 
         noexec = []
         tocheck = set()
@@ -1633,46 +1745,49 @@ class RunQueue:
                     valid_new.add(dep)
 
         invalidtasks = set()
-        for tid in self.rqdata.runtaskentries:
-            if tid not in valid_new and tid not in noexec:
-                invalidtasks.add(tid)
 
-        found = set()
-        processed = set()
-        for tid in invalidtasks:
+        toptasks = set(["{}:{}".format(t[3], t[2]) for t in self.rqdata.targets])
+        for tid in toptasks:
             toprocess = set([tid])
             while toprocess:
                 next = set()
+                visited_invalid = set()
                 for t in toprocess:
-                    for dep in self.rqdata.runtaskentries[t].depends:
-                        if dep in invalidtasks:
-                            found.add(tid)
-                        if dep not in processed:
-                            processed.add(dep)
+                    if t not in valid_new and t not in noexec:
+                        invalidtasks.update(get_root_invalid_tasks(t, self.rqdata.runtaskentries, valid_new, noexec, visited_invalid))
+                        continue
+                    if t in self.rqdata.runq_setscene_tids:
+                        for dep in self.rqexe.sqdata.sq_deps[t]:
                             next.add(dep)
+                        continue
+
+                    for dep in self.rqdata.runtaskentries[t].depends:
+                        next.add(dep)
+
                 toprocess = next
-                if tid in found:
-                    toprocess = set()
 
         tasklist = []
-        for tid in invalidtasks.difference(found):
+        for tid in invalidtasks:
             tasklist.append(tid)
 
         if tasklist:
             bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist))
 
-        return invalidtasks.difference(found)
+        return invalidtasks
 
     def write_diffscenetasks(self, invalidtasks):
+        bb.siggen.check_siggen_version(bb.siggen)
 
         # Define recursion callback
         def recursecb(key, hash1, hash2):
             hashes = [hash1, hash2]
+            bb.debug(1, "Recursively looking for recipe {} hashes {}".format(key, hashes))
             hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData)
+            bb.debug(1, "Found hashfiles:\n{}".format(hashfiles))
 
             recout = []
             if len(hashfiles) == 2:
-                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb)
                 recout.extend(list('    ' + l for l in out2))
             else:
                 recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
@@ -1683,20 +1798,25 @@ class RunQueue:
         for tid in invalidtasks:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            h = self.rqdata.runtaskentries[tid].hash
-            matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
+            h = self.rqdata.runtaskentries[tid].unihash
+            bb.debug(1, "Looking for recipe {} task {}".format(pn, taskname))
+            matches = bb.siggen.find_siginfo(pn, taskname, [], self.cooker.databuilder.mcdata[mc])
+            bb.debug(1, "Found hashfiles:\n{}".format(matches))
             match = None
-            for m in matches:
-                if h in m:
-                    match = m
+            for m in matches.values():
+                if h in m['path']:
+                    match = m['path']
             if match is None:
-                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h)
+                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s tid: %s)?" % (h, tid))
             matches = {k : v for k, v in iter(matches.items()) if h not in k}
+            matches_local = {k : v for k, v in iter(matches.items()) if h not in k and not v['sstate']}
+            if matches_local:
+                matches = matches_local
             if matches:
-                latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1]
+                latestmatch = matches[sorted(matches.keys(), key=lambda h: matches[h]['time'])[-1]]['path']
                 prevh = __find_sha256__.search(latestmatch).group(0)
                 output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb)
-                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, closest matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
+                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, most recent matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
 
 
 class RunQueueExecute:
@@ -1709,6 +1829,10 @@ class RunQueueExecute:
 
         self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1)
         self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
+        self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU")
+        self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO")
+        self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY")
+        self.max_loadfactor = self.cfgData.getVar("BB_LOADFACTOR_MAX")
 
         self.sq_buildable = set()
         self.sq_running = set()
@@ -1726,6 +1850,8 @@ class RunQueueExecute:
         self.build_stamps2 = []
         self.failed_tids = []
         self.sq_deferred = {}
+        self.sq_needed_harddeps = set()
+        self.sq_harddep_deferred = set()
 
         self.stampcache = {}
 
@@ -1733,17 +1859,39 @@ class RunQueueExecute:
         self.holdoff_need_update = True
         self.sqdone = False
 
-        self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
-        self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
-
-        for mc in rq.worker:
-            rq.worker[mc].pipe.setrunqueueexec(self)
-        for mc in rq.fakeworker:
-            rq.fakeworker[mc].pipe.setrunqueueexec(self)
+        self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids))
 
         if self.number_tasks <= 0:
              bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks)
 
+        lower_limit = 1.0
+        upper_limit = 1000000.0
+        if self.max_cpu_pressure:
+            self.max_cpu_pressure = float(self.max_cpu_pressure)
+            if self.max_cpu_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_CPU %s, minimum value is %s." % (self.max_cpu_pressure, lower_limit))
+            if self.max_cpu_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_CPU is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_cpu_pressure))
+
+        if self.max_io_pressure:
+            self.max_io_pressure = float(self.max_io_pressure)
+            if self.max_io_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_IO %s, minimum value is %s." % (self.max_io_pressure, lower_limit))
+            if self.max_io_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
+
+        if self.max_memory_pressure:
+            self.max_memory_pressure = float(self.max_memory_pressure)
+            if self.max_memory_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit))
+            if self.max_memory_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
+
+        if self.max_loadfactor:
+            self.max_loadfactor = float(self.max_loadfactor)
+            if self.max_loadfactor <= 0:
+                bb.fatal("Invalid BB_LOADFACTOR_MAX %s, needs to be greater than zero." % (self.max_loadfactor))
+            
         # List of setscene tasks which we've covered
         self.scenequeue_covered = set()
         # List of tasks which are covered (including setscene ones)
@@ -1753,11 +1901,6 @@ class RunQueueExecute:
         self.tasks_notcovered = set()
         self.scenequeue_notneeded = set()
 
-        # We can't skip specified target tasks which aren't setscene tasks
-        self.cantskip = set(self.rqdata.target_tids)
-        self.cantskip.difference_update(self.rqdata.runq_setscene_tids)
-        self.cantskip.intersection_update(self.rqdata.runtaskentries)
-
         schedulers = self.get_schedulers()
         for scheduler in schedulers:
             if self.scheduler == scheduler.name:
@@ -1768,11 +1911,29 @@ class RunQueueExecute:
             bb.fatal("Invalid scheduler '%s'.  Available schedulers: %s" %
                      (self.scheduler, ", ".join(obj.name for obj in schedulers)))
 
-        #if len(self.rqdata.runq_setscene_tids) > 0:
+        #if self.rqdata.runq_setscene_tids:
         self.sqdata = SQData()
-        build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
+        build_scenequeue_data(self.sqdata, self.rqdata, self)
+
+        update_scenequeue_data(self.sqdata.sq_revdeps, self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=True)
+
+        # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+        # in any stamp files. Pass the list out to metadata as an event.
+        found = {}
+        for tid in self.rqdata.runq_setscene_tids:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            stamps = bb.build.find_stale_stamps(taskname, taskfn)
+            if stamps:
+                if mc not in found:
+                    found[mc] = {}
+                found[mc][tid] = stamps
+        for mc in found:
+            event = bb.event.StaleSetSceneTasks(found[mc])
+            bb.event.fire(event, self.cooker.databuilder.mcdata[mc])
+
+        self.build_taskdepdata_cache()
 
-    def runqueue_process_waitpid(self, task, status):
+    def runqueue_process_waitpid(self, task, status, fakerootlog=None):
 
         # self.build_stamps[pid] may not exist when use shared work directory.
         if task in self.build_stamps:
@@ -1785,9 +1946,10 @@ class RunQueueExecute:
             else:
                 self.sq_task_complete(task)
             self.sq_live.remove(task)
+            self.stats.updateActiveSetscene(len(self.sq_live))
         else:
             if status != 0:
-                self.task_fail(task, status)
+                self.task_fail(task, status, fakerootlog=fakerootlog)
             else:
                 self.task_complete(task)
         return True
@@ -1795,20 +1957,20 @@ class RunQueueExecute:
     def finish_now(self):
         for mc in self.rq.worker:
             try:
-                self.rq.worker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, b"", "finishnow")
                 self.rq.worker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
                 pass
         for mc in self.rq.fakeworker:
             try:
-                self.rq.fakeworker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, b"", "finishnow")
                 self.rq.fakeworker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
                 pass
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return
 
@@ -1818,13 +1980,13 @@ class RunQueueExecute:
     def finish(self):
         self.rq.state = runQueueCleanUp
 
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         if active > 0:
             bb.event.fire(runQueueExitWait(active), self.cfgData)
             self.rq.read_workers()
             return self.rq.active_fds()
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return True
 
@@ -1851,7 +2013,7 @@ class RunQueueExecute:
         return valid
 
     def can_start_task(self):
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         can_start = active < self.number_tasks
         return can_start
 
@@ -1871,8 +2033,7 @@ class RunQueueExecute:
                 try:
                     module = __import__(modname, fromlist=(name,))
                 except ImportError as exc:
-                    logger.critical("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc))
-                    raise SystemExit(1)
+                    bb.fatal("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc))
                 else:
                     schedulers.add(getattr(module, name))
         return schedulers
@@ -1902,21 +2063,52 @@ class RunQueueExecute:
                 self.setbuildable(revdep)
                 logger.debug("Marking task %s as buildable", revdep)
 
+        found = None
+        for t in sorted(self.sq_deferred.copy()):
+            if self.sq_deferred[t] == task:
+                # Allow the next deferred task to run. Any other deferred tasks should be deferred after that task.
+                # We shouldn't allow all to run at once as it is prone to races.
+                if not found:
+                    bb.debug(1, "Deferred task %s now buildable" % t)
+                    del self.sq_deferred[t]
+                    update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+                    found = t
+                else:
+                    bb.debug(1, "Deferring %s after %s" % (t, found))
+                    self.sq_deferred[t] = found
+
     def task_complete(self, task):
         self.stats.taskCompleted()
         bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.task_completeoutright(task)
         self.runq_tasksrun.add(task)
 
-    def task_fail(self, task, exitcode):
+    def task_fail(self, task, exitcode, fakerootlog=None):
         """
         Called when a task has failed
         Updates the state engine with the failure
         """
         self.stats.taskFailed()
         self.failed_tids.append(task)
-        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
-        if self.rqdata.taskData[''].abort:
+
+        fakeroot_log = []
+        if fakerootlog and os.path.exists(fakerootlog):
+            with open(fakerootlog) as fakeroot_log_file:
+                fakeroot_failed = False
+                for line in reversed(fakeroot_log_file.readlines()):
+                    for fakeroot_error in ['mismatch', 'error', 'fatal']:
+                        if fakeroot_error in line.lower():
+                            fakeroot_failed = True
+                    if 'doing new pid setup and server start' in line:
+                        break
+                    fakeroot_log.append(line)
+
+            if not fakeroot_failed:
+                fakeroot_log = []
+
+        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=("".join(fakeroot_log) or None)), self.cfgData)
+
+        if self.rqdata.taskData[''].halt:
             self.rq.state = runQueueCleanUp
 
     def task_skip(self, task, reason):
@@ -1931,7 +2123,7 @@ class RunQueueExecute:
         err = False
         if not self.sqdone:
             logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
-            completeevent = sceneQueueComplete(self.sq_stats, self.rq)
+            completeevent = sceneQueueComplete(self.stats, self.rq)
             bb.event.fire(completeevent, self.cfgData)
         if self.sq_deferred:
             logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred))
@@ -1943,6 +2135,10 @@ class RunQueueExecute:
             logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
             err = True
 
+        for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered):
+            # No task should end up in both covered and uncovered, that is a bug.
+            logger.error("Setscene task %s in both covered and notcovered." % tid)
+
         for tid in self.rqdata.runq_setscene_tids:
             if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
                 err = True
@@ -1961,7 +2157,7 @@ class RunQueueExecute:
             if x not in self.tasks_scenequeue_done:
                 logger.error("Task %s was never processed by the setscene code" % x)
                 err = True
-            if len(self.rqdata.runtaskentries[x].depends) == 0 and x not in self.runq_buildable:
+            if not self.rqdata.runtaskentries[x].depends and x not in self.runq_buildable:
                 logger.error("Task %s was never marked as buildable by the setscene code" % x)
                 err = True
         return err
@@ -1984,8 +2180,11 @@ class RunQueueExecute:
         if not self.sqdone and self.can_start_task():
             # Find the next setscene to run
             for nexttask in self.sorted_setscene_tids:
-                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
-                    if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
+                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
+                    if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
+                            nexttask not in self.sq_needed_harddeps and \
+                            self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
+                            self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
                             logger.debug2("Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
@@ -1993,6 +2192,19 @@ class RunQueueExecute:
                             if nexttask in self.sq_deferred:
                                 del self.sq_deferred[nexttask]
                             return True
+                    if nexttask in self.sqdata.sq_harddeps_rev and not self.sqdata.sq_harddeps_rev[nexttask].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                        logger.debug2("Deferring %s due to hard dependencies" % nexttask)
+                        updated = False
+                        for dep in self.sqdata.sq_harddeps_rev[nexttask]:
+                            if dep not in self.sq_needed_harddeps:
+                                logger.debug2("Enabling task %s as it is a hard dependency" % dep)
+                                self.sq_buildable.add(dep)
+                                self.sq_needed_harddeps.add(dep)
+                                updated = True
+                        self.sq_harddep_deferred.add(nexttask)
+                        if updated:
+                            return True
+                        continue
                     # If covered tasks are running, need to wait for them to complete
                     for t in self.sqdata.sq_covered_tasks[nexttask]:
                         if t in self.runq_running and t not in self.runq_complete:
@@ -2007,8 +2219,6 @@ class RunQueueExecute:
                             logger.debug("%s didn't become valid, skipping setscene" % nexttask)
                             self.sq_task_failoutright(nexttask)
                             return True
-                        else:
-                            self.sqdata.outrightfail.remove(nexttask)
                     if nexttask in self.sqdata.outrightfail:
                         logger.debug2('No package found, so skipping setscene task %s', nexttask)
                         self.sq_task_failoutright(nexttask)
@@ -2040,28 +2250,42 @@ class RunQueueExecute:
                 self.sq_task_failoutright(task)
                 return True
 
-            startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
+            startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
             bb.event.fire(startevent, self.cfgData)
 
-            taskdepdata = self.sq_build_taskdepdata(task)
-
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            taskhash = self.rqdata.get_task_hash(task)
-            unihash = self.rqdata.get_task_unihash(task)
+            realfn = bb.cache.virtualfn2realfn(taskfn)[0]
+            runtask = {
+                'fn' : taskfn,
+                'task' : task,
+                'taskname' : taskname,
+                'taskhash' : self.rqdata.get_task_hash(task),
+                'unihash' : self.rqdata.get_task_unihash(task),
+                'quieterrors' : True,
+                'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+                'layername' : self.cooker.collections[mc].calc_bbfile_priority(realfn)[2],
+                'taskdepdata' : self.sq_build_taskdepdata(task),
+                'dry_run' : False,
+                'taskdep': taskdep,
+                'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+                'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+                'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+            }
+
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
-            self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             self.build_stamps2.append(self.build_stamps[task])
             self.sq_running.add(task)
             self.sq_live.add(task)
-            self.sq_stats.taskActive()
+            self.stats.updateActiveSetscene(len(self.sq_live))
             if self.can_start_task():
                 return True
 
@@ -2092,9 +2316,9 @@ class RunQueueExecute:
         if task is not None:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
 
-            if self.rqdata.setscenewhitelist is not None:
-                if self.check_setscenewhitelist(task):
-                    self.task_fail(task, "setscene whitelist")
+            if self.rqdata.setscene_ignore_tasks is not None:
+                if self.check_setscene_ignore_tasks(task):
+                    self.task_fail(task, "setscene ignore_tasks")
                     return True
 
             if task in self.tasks_covered:
@@ -2117,18 +2341,32 @@ class RunQueueExecute:
                 self.runq_running.add(task)
                 self.stats.taskActive()
                 if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
-                    bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
+                    bb.build.make_stamp_mcfn(taskname, taskfn)
                 self.task_complete(task)
                 return True
             else:
                 startevent = runQueueTaskStarted(task, self.stats, self.rq)
                 bb.event.fire(startevent, self.cfgData)
 
-            taskdepdata = self.build_taskdepdata(task)
-
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            taskhash = self.rqdata.get_task_hash(task)
-            unihash = self.rqdata.get_task_unihash(task)
+            realfn = bb.cache.virtualfn2realfn(taskfn)[0]
+            runtask = {
+                'fn' : taskfn,
+                'task' : task,
+                'taskname' : taskname,
+                'taskhash' : self.rqdata.get_task_hash(task),
+                'unihash' : self.rqdata.get_task_unihash(task),
+                'quieterrors' : False,
+                'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+                'layername' : self.cooker.collections[mc].calc_bbfile_priority(realfn)[2],
+                'taskdepdata' : self.build_taskdepdata(task),
+                'dry_run' : self.rqdata.setscene_enforce,
+                'taskdep': taskdep,
+                'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+                'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+                'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+            }
+
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                 if not mc in self.rq.fakeworker:
                     try:
@@ -2138,31 +2376,31 @@ class RunQueueExecute:
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
-            self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             self.build_stamps2.append(self.build_stamps[task])
             self.runq_running.add(task)
             self.stats.taskActive()
             if self.can_start_task():
                 return True
 
-        if self.stats.active > 0 or self.sq_stats.active > 0:
+        if self.stats.active > 0 or self.sq_live:
             self.rq.read_workers()
             return self.rq.active_fds()
 
         # No more tasks can be run. If we have deferred setscene tasks we should run them.
         if self.sq_deferred:
-            tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
-            logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
-            self.sq_task_failoutright(tid)
+            deferred_tid = list(self.sq_deferred.keys())[0]
+            blocking_tid = self.sq_deferred.pop(deferred_tid)
+            logger.warning("Runqueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid))
             return True
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return True
 
@@ -2195,6 +2433,22 @@ class RunQueueExecute:
             ret.add(dep)
         return ret
 
+    # Build the individual cache entries in advance once to save time
+    def build_taskdepdata_cache(self):
+        taskdepdata_cache = {}
+        for task in self.rqdata.runtaskentries:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
+            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
+            deps = self.rqdata.runtaskentries[task].depends
+            provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
+            taskhash = self.rqdata.runtaskentries[task].hash
+            unihash = self.rqdata.runtaskentries[task].unihash
+            deps = self.filtermcdeps(task, mc, deps)
+            hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
+            taskdepdata_cache[task] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+
+        self.taskdepdata_cache = taskdepdata_cache
+
     # We filter out multiconfig dependencies from taskdepdata we pass to the tasks
     # as most code can't handle them
     def build_taskdepdata(self, task):
@@ -2206,15 +2460,9 @@ class RunQueueExecute:
         while next:
             additional = []
             for revdep in next:
-                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-                deps = self.rqdata.runtaskentries[revdep].depends
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                deps = self.filtermcdeps(task, mc, deps)
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
-                for revdep2 in deps:
+                self.taskdepdata_cache[revdep][6] = self.rqdata.runtaskentries[revdep].unihash
+                taskdepdata[revdep] = self.taskdepdata_cache[revdep]
+                for revdep2 in self.taskdepdata_cache[revdep][3]:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
             next = additional
@@ -2228,7 +2476,7 @@ class RunQueueExecute:
             return
 
         notcovered = set(self.scenequeue_notcovered)
-        notcovered |= self.cantskip
+        notcovered |= self.sqdata.cantskip
         for tid in self.scenequeue_notcovered:
             notcovered |= self.sqdata.sq_covered_tasks[tid]
         notcovered |= self.sqdata.unskippable.difference(self.rqdata.runq_setscene_tids)
@@ -2241,7 +2489,7 @@ class RunQueueExecute:
         covered.intersection_update(self.tasks_scenequeue_done)
 
         for tid in notcovered | covered:
-            if len(self.rqdata.runtaskentries[tid].depends) == 0:
+            if not self.rqdata.runtaskentries[tid].depends:
                 self.setbuildable(tid)
             elif self.rqdata.runtaskentries[tid].depends.issubset(self.runq_complete):
                  self.setbuildable(tid)
@@ -2273,10 +2521,19 @@ class RunQueueExecute:
             self.updated_taskhash_queue.remove((tid, unihash))
 
             if unihash != self.rqdata.runtaskentries[tid].unihash:
-                hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
-                self.rqdata.runtaskentries[tid].unihash = unihash
-                bb.parse.siggen.set_unihash(tid, unihash)
-                toprocess.add(tid)
+                # Make sure we rehash any other tasks with the same task hash that we're deferred against.
+                torehash = [tid]
+                for deftid in self.sq_deferred:
+                    if self.sq_deferred[deftid] == tid:
+                        torehash.append(deftid)
+                for hashtid in torehash:
+                    hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash))
+                    self.rqdata.runtaskentries[hashtid].unihash = unihash
+                    bb.parse.siggen.set_unihash(hashtid, unihash)
+                    toprocess.add(hashtid)
+                if torehash:
+                    # Need to save after set_unihash above
+                    bb.parse.siggen.save_unitaskhashes()
 
         # Work out all tasks which depend upon these
         total = set()
@@ -2294,7 +2551,7 @@ class RunQueueExecute:
         # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
         next = set()
         for p in total:
-            if len(self.rqdata.runtaskentries[p].depends) == 0:
+            if not self.rqdata.runtaskentries[p].depends:
                 next.add(p)
             elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
                 next.add(p)
@@ -2304,11 +2561,10 @@ class RunQueueExecute:
             current = next.copy()
             next = set()
             for tid in current:
-                if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
+                if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
                 orighash = self.rqdata.runtaskentries[tid].hash
-                dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid))
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, dc)
+                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
                 origuni = self.rqdata.runtaskentries[tid].unihash
                 newuni = bb.parse.siggen.get_unihash(tid)
                 # FIXME, need to check it can come from sstate at all for determinism?
@@ -2334,9 +2590,9 @@ class RunQueueExecute:
 
         if changed:
             for mc in self.rq.worker:
-                self.rq.worker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
             for mc in self.rq.fakeworker:
-                self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
 
             hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed)))
 
@@ -2370,7 +2626,7 @@ class RunQueueExecute:
                 self.tasks_scenequeue_done.remove(tid)
             for dep in self.sqdata.sq_covered_tasks[tid]:
                 if dep in self.runq_complete and dep not in self.runq_tasksrun:
-                    bb.error("Task %s marked as completed but now needing to rerun? Aborting build." % dep)
+                    bb.error("Task %s marked as completed but now needing to rerun? Halting build." % dep)
                     self.failed_tids.append(tid)
                     self.rq.state = runQueueCleanUp
                     return
@@ -2383,17 +2639,6 @@ class RunQueueExecute:
                 self.sq_buildable.remove(tid)
             if tid in self.sq_running:
                 self.sq_running.remove(tid)
-            harddepfail = False
-            for t in self.sqdata.sq_harddeps:
-                if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
-                    harddepfail = True
-                    break
-            if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
-                if tid not in self.sq_buildable:
-                    self.sq_buildable.add(tid)
-            if len(self.sqdata.sq_revdeps[tid]) == 0:
-                self.sq_buildable.add(tid)
-
             if tid in self.sqdata.outrightfail:
                 self.sqdata.outrightfail.remove(tid)
             if tid in self.scenequeue_notcovered:
@@ -2404,7 +2649,7 @@ class RunQueueExecute:
                 self.scenequeue_notneeded.remove(tid)
 
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-            self.sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
 
             if tid in self.stampcache:
                 del self.stampcache[tid]
@@ -2412,28 +2657,62 @@ class RunQueueExecute:
             if tid in self.build_stamps:
                 del self.build_stamps[tid]
 
-            update_tasks.append((tid, harddepfail, tid in self.sqdata.valid))
+            update_tasks.append(tid)
+
+        update_tasks2 = []
+        for tid in update_tasks:
+            harddepfail = False
+            for t in self.sqdata.sq_harddeps_rev[tid]:
+                if t in self.scenequeue_notcovered:
+                    harddepfail = True
+                    break
+            if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                if tid not in self.sq_buildable:
+                    self.sq_buildable.add(tid)
+            if not self.sqdata.sq_revdeps[tid]:
+                self.sq_buildable.add(tid)
 
-        if update_tasks:
+            update_tasks2.append((tid, harddepfail, tid in self.sqdata.valid))
+
+        if update_tasks2:
             self.sqdone = False
-            update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+            for mc in sorted(self.sqdata.multiconfigs):
+                for tid in sorted([t[0] for t in update_tasks2]):
+                    if mc_from_tid(tid) != mc:
+                        continue
+                    h = pending_hash_index(tid, self.rqdata)
+                    if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
+                        self.sq_deferred[tid] = self.sqdata.hashes[h]
+                        bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
+            update_scenequeue_data([t[0] for t in update_tasks2], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
 
-        for (tid, harddepfail, origvalid) in update_tasks:
+        for (tid, harddepfail, origvalid) in update_tasks2:
             if tid in self.sqdata.valid and not origvalid:
                 hashequiv_logger.verbose("Setscene task %s became valid" % tid)
             if harddepfail:
+                logger.debug2("%s has an unavailable hard dependency so skipping" % (tid))
                 self.sq_task_failoutright(tid)
 
         if changed:
+            self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
+            self.sq_needed_harddeps = set()
+            self.sq_harddep_deferred = set()
             self.holdoff_need_update = True
 
     def scenequeue_updatecounters(self, task, fail=False):
 
-        for dep in sorted(self.sqdata.sq_deps[task]):
-            if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
+        if fail and task in self.sqdata.sq_harddeps:
+            for dep in sorted(self.sqdata.sq_harddeps[task]):
+                if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
+                    # dependency could be already processed, e.g. noexec setscene task
+                    continue
+                noexec, stamppresent = check_setscene_stamps(dep, self.rqdata, self.rq, self.stampcache)
+                if noexec or stamppresent:
+                    continue
                 logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
+        for dep in sorted(self.sqdata.sq_deps[task]):
             if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
                 if dep not in self.sq_buildable:
                     self.sq_buildable.add(dep)
@@ -2452,6 +2731,14 @@ class RunQueueExecute:
                         new.add(dep)
             next = new
 
+        # If this task was one which other setscene tasks have a hard dependency upon, we need
+        # to walk through the hard dependencies and allow execution of those which have completed dependencies.
+        if task in self.sqdata.sq_harddeps:
+            for dep in self.sq_harddep_deferred.copy():
+                if self.sqdata.sq_harddeps_rev[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                    self.sq_harddep_deferred.remove(dep)
+
+        self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
         self.holdoff_need_update = True
 
     def sq_task_completeoutright(self, task):
@@ -2466,22 +2753,20 @@ class RunQueueExecute:
         self.scenequeue_updatecounters(task)
 
     def sq_check_taskfail(self, task):
-        if self.rqdata.setscenewhitelist is not None:
+        if self.rqdata.setscene_ignore_tasks is not None:
             realtask = task.split('_setscene')[0]
             (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask)
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+            if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks):
                 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
                 self.rq.state = runQueueCleanUp
 
     def sq_task_complete(self, task):
-        self.sq_stats.taskCompleted()
-        bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData)
+        bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.sq_task_completeoutright(task)
 
     def sq_task_fail(self, task, result):
-        self.sq_stats.taskFailed()
-        bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
+        bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
         self.sq_check_taskfail(task)
@@ -2489,8 +2774,6 @@ class RunQueueExecute:
     def sq_task_failoutright(self, task):
         self.sq_running.add(task)
         self.sq_buildable.add(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
 
@@ -2498,8 +2781,6 @@ class RunQueueExecute:
         self.sq_running.add(task)
         self.sq_buildable.add(task)
         self.sq_task_completeoutright(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
 
     def sq_build_taskdepdata(self, task):
         def getsetscenedeps(tid):
@@ -2530,7 +2811,8 @@ class RunQueueExecute:
                 provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
                 taskhash = self.rqdata.runtaskentries[revdep].hash
                 unihash = self.rqdata.runtaskentries[revdep].unihash
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
+                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
+                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -2539,8 +2821,8 @@ class RunQueueExecute:
         #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
         return taskdepdata
 
-    def check_setscenewhitelist(self, tid):
-        # Check task that is going to run against the whitelist
+    def check_setscene_ignore_tasks(self, tid):
+        # Check task that is going to run against the ignore tasks list
         (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
         # Ignore covered tasks
         if tid in self.tasks_covered:
@@ -2554,14 +2836,15 @@ class RunQueueExecute:
             return False
 
         pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-        if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+        if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks):
             if tid in self.rqdata.runq_setscene_tids:
-                msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
+                msg = ['Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)]
             else:
-                msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
+                msg = ['Task %s.%s attempted to execute unexpectedly' % (pn, taskname)]
             for t in self.scenequeue_notcovered:
-                msg = msg + "\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)
-            logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
+                msg.append("\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash))
+            msg.append('\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
+            logger.error("".join(msg))
             return True
         return False
 
@@ -2573,6 +2856,7 @@ class SQData(object):
         self.sq_revdeps = {}
         # Injected inter-setscene task dependencies
         self.sq_harddeps = {}
+        self.sq_harddeps_rev = {}
         # Cache of stamp files so duplicates can't run in parallel
         self.stamps = {}
         # Setscene tasks directly depended upon by the build
@@ -2582,12 +2866,17 @@ class SQData(object):
         # A list of normal tasks a setscene task covers
         self.sq_covered_tasks = {}
 
-def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
+def build_scenequeue_data(sqdata, rqdata, sqrq):
 
     sq_revdeps = {}
     sq_revdeps_squash = {}
     sq_collated_deps = {}
 
+    # We can't skip specified target tasks which aren't setscene tasks
+    sqdata.cantskip = set(rqdata.target_tids)
+    sqdata.cantskip.difference_update(rqdata.runq_setscene_tids)
+    sqdata.cantskip.intersection_update(rqdata.runtaskentries)
+
     # We need to construct a dependency graph for the setscene functions. Intermediate
     # dependencies between the setscene tasks only complicate the code. This code
     # therefore aims to collapse the huge runqueue dependency tree into a smaller one
@@ -2600,7 +2889,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     for tid in rqdata.runtaskentries:
         sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps)
         sq_revdeps_squash[tid] = set()
-        if (len(sq_revdeps[tid]) == 0) and tid not in rqdata.runq_setscene_tids:
+        if not sq_revdeps[tid] and tid not in rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint %s" % (tid))
             endpoints[tid] = set()
 
@@ -2634,16 +2923,15 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             sq_revdeps_squash[point] = set()
             if point in rqdata.runq_setscene_tids:
                 sq_revdeps_squash[point] = tasks
-                tasks = set()
                 continue
             for dep in rqdata.runtaskentries[point].depends:
                 if point in sq_revdeps[dep]:
                     sq_revdeps[dep].remove(point)
                 if tasks:
                     sq_revdeps_squash[dep] |= tasks
-                if len(sq_revdeps[dep]) == 0 and dep not in rqdata.runq_setscene_tids:
+                if not sq_revdeps[dep] and dep not in rqdata.runq_setscene_tids:
                     newendpoints[dep] = task
-        if len(newendpoints) != 0:
+        if newendpoints:
             process_endpoints(newendpoints)
 
     process_endpoints(endpoints)
@@ -2655,16 +2943,16 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     # Take the build endpoints (no revdeps) and find the sstate tasks they depend upon
     new = True
     for tid in rqdata.runtaskentries:
-        if len(rqdata.runtaskentries[tid].revdeps) == 0:
+        if not rqdata.runtaskentries[tid].revdeps:
             sqdata.unskippable.add(tid)
-    sqdata.unskippable |= sqrq.cantskip
+    sqdata.unskippable |= sqdata.cantskip
     while new:
         new = False
         orig = sqdata.unskippable.copy()
         for tid in sorted(orig, reverse=True):
             if tid in rqdata.runq_setscene_tids:
                 continue
-            if len(rqdata.runtaskentries[tid].depends) == 0:
+            if not rqdata.runtaskentries[tid].depends:
                 # These are tasks which have no setscene tasks in their chain, need to mark as directly buildable
                 sqrq.setbuildable(tid)
             sqdata.unskippable |= rqdata.runtaskentries[tid].depends
@@ -2679,8 +2967,8 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     for taskcounter, tid in enumerate(rqdata.runtaskentries):
         if tid in rqdata.runq_setscene_tids:
             pass
-        elif len(sq_revdeps_squash[tid]) != 0:
-            bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
+        elif sq_revdeps_squash[tid]:
+            bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, halting. Please report this problem.")
         else:
             del sq_revdeps_squash[tid]
         rqdata.init_progress_reporter.update(taskcounter)
@@ -2694,7 +2982,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
         (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
         realtid = tid + "_setscene"
         idepends = rqdata.taskData[mc].taskentries[realtid].idepends
-        sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True)
+        sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
+
+        sqdata.sq_harddeps_rev[tid] = set()
         for (depname, idependtask) in idepends:
 
             if depname not in rqdata.taskData[mc].build_targets:
@@ -2707,20 +2997,15 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             if deptid not in rqdata.runtaskentries:
                 bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
 
+            logger.debug2("Adding hard setscene dependency %s for %s" % (deptid, tid))
+
             if not deptid in sqdata.sq_harddeps:
                 sqdata.sq_harddeps[deptid] = set()
             sqdata.sq_harddeps[deptid].add(tid)
-
-            sq_revdeps_squash[tid].add(deptid)
-            # Have to zero this to avoid circular dependencies
-            sq_revdeps_squash[deptid] = set()
+            sqdata.sq_harddeps_rev[tid].add(deptid)
 
     rqdata.init_progress_reporter.next_stage()
 
-    for task in sqdata.sq_harddeps:
-        for dep in sqdata.sq_harddeps[task]:
-            sq_revdeps_squash[dep].add(task)
-
     rqdata.init_progress_reporter.next_stage()
 
     #for tid in sq_revdeps_squash:
@@ -2744,16 +3029,47 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     sqdata.multiconfigs = set()
     for tid in sqdata.sq_revdeps:
         sqdata.multiconfigs.add(mc_from_tid(tid))
-        if len(sqdata.sq_revdeps[tid]) == 0:
+        if not sqdata.sq_revdeps[tid]:
             sqrq.sq_buildable.add(tid)
 
-    rqdata.init_progress_reporter.finish()
+    rqdata.init_progress_reporter.next_stage()
 
     sqdata.noexec = set()
     sqdata.stamppresent = set()
     sqdata.valid = set()
 
-    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
+    sqdata.hashes = {}
+    sqrq.sq_deferred = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
+            if mc_from_tid(tid) != mc:
+                continue
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.debug(1, "Deferring %s after %s" % (tid, sqdata.hashes[h]))
+
+def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False):
+
+    (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+
+    taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+
+    if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        bb.build.make_stamp_mcfn(taskname + "_setscene", taskfn)
+        return True, False
+
+    if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+        logger.debug2('Setscene stamp current for task %s', tid)
+        return False, True
+
+    if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+        logger.debug2('Normal stamp current for task %s', tid)
+        return False, True
+
+    return False, False
 
 def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
 
@@ -2764,55 +3080,42 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
             sqdata.stamppresent.remove(tid)
         if tid in sqdata.valid:
             sqdata.valid.remove(tid)
+        if tid in sqdata.outrightfail:
+            sqdata.outrightfail.remove(tid)
 
-        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-
-        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+        noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True)
 
-        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        if noexec:
             sqdata.noexec.add(tid)
             sqrq.sq_task_skip(tid)
-            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
-            continue
-
-        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-            logger.debug2('Setscene stamp current for task %s', tid)
-            sqdata.stamppresent.add(tid)
-            sqrq.sq_task_skip(tid)
+            logger.debug2("%s is noexec so skipping setscene" % (tid))
             continue
 
-        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-            logger.debug2('Normal stamp current for task %s', tid)
+        if stamppresent:
             sqdata.stamppresent.add(tid)
             sqrq.sq_task_skip(tid)
+            logger.debug2("%s has a valid stamp, skipping" % (tid))
             continue
 
         tocheck.add(tid)
 
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
-    sqdata.hashes = {}
-    for mc in sorted(sqdata.multiconfigs):
-        for tid in sorted(sqdata.sq_revdeps):
-            if mc_from_tid(tid) != mc:
-                continue
-            if tid in sqdata.stamppresent:
-                continue
-            if tid in sqdata.valid:
-                continue
-            if tid in sqdata.noexec:
-                continue
-            if tid in sqrq.scenequeue_notcovered:
-                continue
-            sqdata.outrightfail.add(tid)
-
-            h = pending_hash_index(tid, rqdata)
-            if h not in sqdata.hashes:
-                sqdata.hashes[h] = tid
-            else:
-                sqrq.sq_deferred[tid] = sqdata.hashes[h]
-                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
-
+    for tid in tids:
+        if tid in sqdata.stamppresent:
+            continue
+        if tid in sqdata.valid:
+            continue
+        if tid in sqdata.noexec:
+            continue
+        if tid in sqrq.scenequeue_covered:
+            continue
+        if tid in sqrq.scenequeue_notcovered:
+            continue
+        if tid in sqrq.sq_deferred:
+            continue
+        sqdata.outrightfail.add(tid)
+        logger.debug2("%s already handled (fallthrough), skipping" % (tid))
 
 class TaskFailure(Exception):
     """
@@ -2876,12 +3179,16 @@ class runQueueTaskFailed(runQueueEvent):
     """
     Event notifying a task failed
     """
-    def __init__(self, task, stats, exitcode, rq):
+    def __init__(self, task, stats, exitcode, rq, fakeroot_log=None):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
+        self.fakeroot_log = fakeroot_log
 
     def __str__(self):
-        return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+        if self.fakeroot_log:
+            return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log)
+        else:
+            return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
 
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
@@ -2933,18 +3240,16 @@ class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
     """
-    def __init__(self, pipein, pipeout, d, rq, rqexec):
+    def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None):
         self.input = pipein
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
         self.d = d
         self.rq = rq
         self.rqexec = rqexec
-
-    def setrunqueueexec(self, rqexec):
-        self.rqexec = rqexec
+        self.fakerootlogs = fakerootlogs
 
     def read(self):
         for workers, name in [(self.rq.worker, "Worker"), (self.rq.fakeworker, "Fakeroot")]:
@@ -2956,13 +3261,13 @@ class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue = self.queue + (self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(102400) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
         end = len(self.queue)
         found = True
-        while found and len(self.queue):
+        while found and self.queue:
             found = False
             index = self.queue.find(b"</event>")
             while index != -1 and self.queue.startswith(b"<event>"):
@@ -2987,7 +3292,11 @@ class runQueuePipe():
                     task, status = pickle.loads(self.queue[10:index])
                 except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
-                self.rqexec.runqueue_process_waitpid(task, status)
+                (_, _, _, taskfn) = split_tid_mcfn(task)
+                fakerootlog = None
+                if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs:
+                    fakerootlog = self.fakerootlogs[taskfn]
+                self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog)
                 found = True
                 self.queue = self.queue[index+11:]
                 index = self.queue.find(b"</exitcode>")
@@ -2996,16 +3305,16 @@ class runQueuePipe():
     def close(self):
         while self.read():
             continue
-        if len(self.queue) > 0:
+        if self.queue:
             print("Warning, worker left partial message: %s" % self.queue)
         self.input.close()
 
-def get_setscene_enforce_whitelist(d, targets):
+def get_setscene_enforce_ignore_tasks(d, targets):
     if d.getVar('BB_SETSCENE_ENFORCE') != '1':
         return None
-    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split()
+    ignore_tasks = (d.getVar("BB_SETSCENE_ENFORCE_IGNORE_TASKS") or "").split()
     outlist = []
-    for item in whitelist[:]:
+    for item in ignore_tasks[:]:
         if item.startswith('%:'):
             for (mc, target, task, fn) in targets:
                 outlist.append(target + ':' + item.split(':')[1])
@@ -3013,12 +3322,12 @@ def get_setscene_enforce_whitelist(d, targets):
             outlist.append(item)
     return outlist
 
-def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+def check_setscene_enforce_ignore_tasks(pn, taskname, ignore_tasks):
     import fnmatch
-    if whitelist is not None:
+    if ignore_tasks is not None:
         item = '%s:%s' % (pn, taskname)
-        for whitelist_item in whitelist:
-            if fnmatch.fnmatch(item, whitelist_item):
+        for ignore_tasks in ignore_tasks:
+            if fnmatch.fnmatch(item, ignore_tasks):
                 return True
         return False
     return True
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index b27b4aefe0..76b189291d 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -26,6 +26,9 @@ import errno
 import re
 import datetime
 import pickle
+import traceback
+import gc
+import stat
 import bb.server.xmlrpcserver
 from bb import daemonize
 from multiprocessing import queues
@@ -35,9 +38,46 @@ logger = logging.getLogger('BitBake')
 class ProcessTimeout(SystemExit):
     pass
 
+def currenttime():
+    return datetime.datetime.now().strftime('%H:%M:%S.%f')
+
 def serverlog(msg):
-    print(str(os.getpid()) + " " +  datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
-    sys.stdout.flush()
+    print(str(os.getpid()) + " " +  currenttime() + " " + msg)
+    #Seems a flush here triggers filesytem sync like behaviour and long hangs in the server
+    #sys.stdout.flush()
+
+#
+# When we have lockfile issues, try and find infomation about which process is
+# using the lockfile
+#
+def get_lockfile_process_msg(lockfile):
+    # Some systems may not have lsof available
+    procs = None
+    try:
+        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        # File was deleted?
+        pass
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    if procs is None:
+        # Fall back to fuser if lsof is unavailable
+        try:
+            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError:
+            # File was deleted?
+            pass
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+    if procs:
+        return procs.decode("utf-8")
+    return None
+
+class idleFinish():
+    def __init__(self, msg):
+         self.msg = msg
 
 class ProcessServer():
     profile_filename = "profile.log"
@@ -56,12 +96,19 @@ class ProcessServer():
         self.maxuiwait = 30
         self.xmlrpc = False
 
+        self.idle = None
+        # Need a lock for _idlefuns changes
         self._idlefuns = {}
+        self._idlefuncsLock = threading.Lock()
+        self.idle_cond = threading.Condition(self._idlefuncsLock)
 
         self.bitbake_lock = lock
         self.bitbake_lock_name = lockname
         self.sock = sock
         self.sockname = sockname
+        # It is possible the directory may be renamed. Cache the inode of the socket file
+        # so we can tell if things changed.
+        self.sockinode = os.stat(self.sockname)[stat.ST_INO]
 
         self.server_timeout = server_timeout
         self.timeout = self.server_timeout
@@ -70,7 +117,9 @@ class ProcessServer():
     def register_idle_function(self, function, data):
         """Register a function to be called while the server is idle"""
         assert hasattr(function, '__call__')
-        self._idlefuns[function] = data
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            self._idlefuns[function] = data
+        serverlog("Registering idle function %s" % str(function))
 
     def run(self):
 
@@ -109,6 +158,31 @@ class ProcessServer():
 
         return ret
 
+    def _idle_check(self):
+        return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None
+
+    def wait_for_idle(self, timeout=30):
+        # Wait for the idle loop to have cleared
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            return self.idle_cond.wait_for(self._idle_check, timeout) is not False
+
+    def set_async_cmd(self, cmd):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            ret = self.idle_cond.wait_for(self._idle_check, 30)
+            if ret is False:
+                return False
+            self.cooker.command.currentAsyncCommand = cmd
+            return True
+
+    def clear_async_cmd(self):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            self.cooker.command.currentAsyncCommand = None
+            self.idle_cond.notify_all()
+
+    def get_async_cmd(self):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            return self.cooker.command.currentAsyncCommand
+
     def main(self):
         self.cooker.pre_serve()
 
@@ -123,14 +197,19 @@ class ProcessServer():
             fds.append(self.xmlrpc)
         seendata = False
         serverlog("Entering server connection loop")
+        serverlog("Lockfile is: %s\nSocket is %s (%s)" % (self.bitbake_lock_name, self.sockname, os.path.exists(self.sockname)))
 
         def disconnect_client(self, fds):
-            serverlog("Disconnecting Client")
+            serverlog("Disconnecting Client (socket: %s)" % os.path.exists(self.sockname))
             if self.controllersock:
                 fds.remove(self.controllersock)
                 self.controllersock.close()
                 self.controllersock = False
             if self.haveui:
+                # Wait for the idle loop to have cleared (30s max)
+                if not self.wait_for_idle(30):
+                    serverlog("Idle loop didn't finish queued commands after 30s, exiting.")
+                    self.quit = True
                 fds.remove(self.command_channel)
                 bb.event.unregister_UIHhandler(self.event_handle, True)
                 self.command_channel_reply.writer.close()
@@ -142,12 +221,12 @@ class ProcessServer():
                 self.cooker.clientComplete()
                 self.haveui = False
             ready = select.select(fds,[],[],0)[0]
-            if newconnections:
+            if newconnections and not self.quit:
                 serverlog("Starting new client")
                 conn = newconnections.pop(-1)
                 fds.append(conn)
                 self.controllersock = conn
-            elif self.timeout is None and not ready:
+            elif not self.timeout and not ready:
                 serverlog("No timeout, exiting.")
                 self.quit = True
 
@@ -214,11 +293,14 @@ class ProcessServer():
                     continue
                 try:
                     serverlog("Running command %s" % command)
-                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
-                    serverlog("Command Completed")
+                    reply = self.cooker.command.runCommand(command, self)
+                    serverlog("Sending reply %s" % repr(reply))
+                    self.command_channel_reply.send(reply)
+                    serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname))
                 except Exception as e:
-                   serverlog('Exception in server main event loop running command %s (%s)' % (command, str(e)))
-                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+                   stack = traceback.format_exc()
+                   serverlog('Exception in server main event loop running command %s (%s)' % (command, stack))
+                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, stack))
 
             if self.xmlrpc in ready:
                 self.xmlrpc.handle_requests()
@@ -241,19 +323,25 @@ class ProcessServer():
 
             ready = self.idle_commands(.1, fds)
 
-        if len(threading.enumerate()) != 1:
-            serverlog("More than one thread left?: " + str(threading.enumerate()))
+        if self.idle:
+            self.idle.join()
 
-        serverlog("Exiting")
+        serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname))
         # Remove the socket file so we don't get any more connections to avoid races
+        # The build directory could have been renamed so if the file isn't the one we created
+        # we shouldn't delete it.
         try:
-            os.unlink(self.sockname)
-        except:
-            pass
+            sockinode = os.stat(self.sockname)[stat.ST_INO]
+            if sockinode == self.sockinode:
+                os.unlink(self.sockname)
+            else:
+                serverlog("bitbake.sock inode mismatch (%s vs %s), not deleting." % (sockinode, self.sockinode))
+        except Exception as err:
+            serverlog("Removing socket file '%s' failed (%s)" % (self.sockname, err))
         self.sock.close()
 
         try:
-            self.cooker.shutdown(True)
+            self.cooker.shutdown(True, idle=False)
             self.cooker.notifier.stop()
             self.cooker.confignotifier.stop()
         except:
@@ -261,6 +349,9 @@ class ProcessServer():
 
         self.cooker.post_serve()
 
+        if len(threading.enumerate()) != 1:
+            serverlog("More than one thread left?: " + str(threading.enumerate()))
+
         # Flush logs before we release the lock
         sys.stdout.flush()
         sys.stderr.flush()
@@ -276,20 +367,21 @@ class ProcessServer():
             except FileNotFoundError:
                 return None
 
-        lockcontents = get_lock_contents(lockfile)
-        serverlog("Original lockfile contents: " + str(lockcontents))
-
         lock.close()
         lock = None
 
         while not lock:
             i = 0
             lock = None
+            if not os.path.exists(os.path.basename(lockfile)):
+                serverlog("Lockfile directory gone, exiting.")
+                return
+
             while not lock and i < 30:
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
                 if not lock:
                     newlockcontents = get_lock_contents(lockfile)
-                    if newlockcontents != lockcontents:
+                    if not newlockcontents[0].startswith([f"{os.getpid()}\n", f"{os.getpid()} "]):
                         # A new server was started, the lockfile contents changed, we can exit
                         serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
                         return
@@ -303,75 +395,108 @@ class ProcessServer():
                 return
 
             if not lock:
-                # Some systems may not have lsof available
-                procs = None
+                procs = get_lockfile_process_msg(lockfile)
+                msg = ["Delaying shutdown due to active processes which appear to be holding bitbake.lock"]
+                if procs:
+                    msg.append(":\n%s" % procs)
+                serverlog("".join(msg))
+
+    def idle_thread(self):
+        if self.cooker.configuration.profile:
+            try:
+                import cProfile as profile
+            except:
+                import profile
+            prof = profile.Profile()
+
+            ret = profile.Profile.runcall(prof, self.idle_thread_internal)
+
+            prof.dump_stats("profile-mainloop.log")
+            bb.utils.process_profilelog("profile-mainloop.log")
+            serverlog("Raw profiling information saved to profile-mainloop.log and processed statistics to profile-mainloop.log.processed")
+        else:
+            self.idle_thread_internal()
+
+    def idle_thread_internal(self):
+        def remove_idle_func(function):
+            with bb.utils.lock_timeout(self._idlefuncsLock):
+                del self._idlefuns[function]
+                self.idle_cond.notify_all()
+
+        while not self.quit:
+            nextsleep = 0.1
+            fds = []
+
+            with bb.utils.lock_timeout(self._idlefuncsLock):
+                items = list(self._idlefuns.items())
+
+            for function, data in items:
                 try:
-                    procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
-                except subprocess.CalledProcessError:
-                    # File was deleted?
-                    continue
-                except OSError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                if procs is None:
-                    # Fall back to fuser if lsof is unavailable
-                    try:
-                        procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
-                    except subprocess.CalledProcessError:
-                        # File was deleted?
+                    retval = function(self, data, False)
+                    if isinstance(retval, idleFinish):
+                        serverlog("Removing idle function %s at idleFinish" % str(function))
+                        remove_idle_func(function)
+                        self.cooker.command.finishAsyncCommand(retval.msg)
+                        nextsleep = None
+                    elif retval is False:
+                        serverlog("Removing idle function %s" % str(function))
+                        remove_idle_func(function)
+                        nextsleep = None
+                    elif retval is True:
+                        nextsleep = None
+                    elif isinstance(retval, float) and nextsleep:
+                        if (retval < nextsleep):
+                            nextsleep = retval
+                    elif nextsleep is None:
                         continue
-                    except OSError as e:
-                        if e.errno != errno.ENOENT:
-                            raise
+                    else:
+                        fds = fds + retval
+                except SystemExit:
+                    raise
+                except Exception as exc:
+                    if not isinstance(exc, bb.BBHandledException):
+                        logger.exception('Running idle function')
+                    remove_idle_func(function)
+                    serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc())
+                    self.quit = True
 
-                msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
-                if procs:
-                    msg += ":\n%s" % str(procs.decode("utf-8"))
-                serverlog(msg)
+            # Create new heartbeat event?
+            now = time.time()
+            if bb.event._heartbeat_enabled and now >= self.next_heartbeat:
+                # We might have missed heartbeats. Just trigger once in
+                # that case and continue after the usual delay.
+                self.next_heartbeat += self.heartbeat_seconds
+                if self.next_heartbeat <= now:
+                    self.next_heartbeat = now + self.heartbeat_seconds
+                if hasattr(self.cooker, "data"):
+                    heartbeat = bb.event.HeartbeatEvent(now)
+                    try:
+                        bb.event.fire(heartbeat, self.cooker.data)
+                    except Exception as exc:
+                        if not isinstance(exc, bb.BBHandledException):
+                            logger.exception('Running heartbeat function')
+                        serverlog("Exception %s broke in idle_thread, exiting" % traceback.format_exc())
+                        self.quit = True
+            if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat:
+                # Shorten timeout so that we we wake up in time for
+                # the heartbeat.
+                nextsleep = self.next_heartbeat - now
+
+            if nextsleep is not None:
+                select.select(fds,[],[],nextsleep)[0]
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
         if not fds:
             fds = []
 
-        for function, data in list(self._idlefuns.items()):
-            try:
-                retval = function(self, data, False)
-                if retval is False:
-                    del self._idlefuns[function]
-                    nextsleep = None
-                elif retval is True:
-                    nextsleep = None
-                elif isinstance(retval, float) and nextsleep:
-                    if (retval < nextsleep):
-                        nextsleep = retval
-                elif nextsleep is None:
-                    continue
-                else:
-                    fds = fds + retval
-            except SystemExit:
-                raise
-            except Exception as exc:
-                if not isinstance(exc, bb.BBHandledException):
-                    logger.exception('Running idle function')
-                del self._idlefuns[function]
-                self.quit = True
-
-        # Create new heartbeat event?
-        now = time.time()
-        if now >= self.next_heartbeat:
-            # We might have missed heartbeats. Just trigger once in
-            # that case and continue after the usual delay.
-            self.next_heartbeat += self.heartbeat_seconds
-            if self.next_heartbeat <= now:
-                self.next_heartbeat = now + self.heartbeat_seconds
-            if hasattr(self.cooker, "data"):
-                heartbeat = bb.event.HeartbeatEvent(now)
-                bb.event.fire(heartbeat, self.cooker.data)
-        if nextsleep and now + nextsleep > self.next_heartbeat:
-            # Shorten timeout so that we we wake up in time for
-            # the heartbeat.
-            nextsleep = self.next_heartbeat - now
+        if not self.idle:
+            self.idle = threading.Thread(target=self.idle_thread)
+            self.idle.start()
+        elif self.idle and not self.idle.is_alive():
+            serverlog("Idle thread terminated, main thread exiting too")
+            bb.error("Idle thread terminated, main thread exiting too")
+            self.quit = True
 
         if nextsleep is not None:
             if self.xmlrpc:
@@ -391,12 +516,18 @@ class ServerCommunicator():
         self.recv = recv
 
     def runCommand(self, command):
-        self.connection.send(command)
+        try:
+            self.connection.send(command)
+        except BrokenPipeError as e:
+            raise BrokenPipeError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         if not self.recv.poll(30):
-            logger.info("No reply from server in 30s")
+            logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime()))
             if not self.recv.poll(30):
-                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
-        ret, exc = self.recv.get()
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime())
+        try:
+            ret, exc = self.recv.get()
+        except EOFError as e:
+            raise EOFError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         # Should probably turn all exceptions in exc back into exceptions?
         # For now, at least handle BBHandledException
         if exc and ("BBHandledException" in exc or "SystemExit" in exc):
@@ -429,6 +560,7 @@ class BitBakeProcessServerConnection(object):
         self.socket_connection = sock
 
     def terminate(self):
+        self.events.close()
         self.socket_connection.close()
         self.connection.connection.close()
         self.connection.recv.close()
@@ -439,13 +571,14 @@ start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
 
 class BitBakeServer(object):
 
-    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
+    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface, profile):
 
         self.server_timeout = server_timeout
         self.xmlrpcinterface = xmlrpcinterface
         self.featureset = featureset
         self.sockname = sockname
         self.bitbake_lock = lock
+        self.profile = profile
         self.readypipe, self.readypipein = os.pipe()
 
         # Place the log in the builddirectory alongside the lock file
@@ -466,7 +599,7 @@ class BitBakeServer(object):
             try:
                 r = ready.get()
             except EOFError:
-                # Trap the child exitting/closing the pipe and error out
+                # Trap the child exiting/closing the pipe and error out
                 r = None
         if not r or r[0] != "r":
             ready.close()
@@ -509,9 +642,9 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, sys.executable, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
-def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
+def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):
 
     import bb.cookerdata
     import bb.cooker
@@ -523,6 +656,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         # Create server control socket
         if os.path.exists(sockname):
+            serverlog("WARNING: removing existing socket file '%s'" % sockname)
             os.unlink(sockname)
 
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -539,7 +673,8 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
         writer = ConnectionWriter(readypipeinfd)
         try:
             featureset = []
-            cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
+            cooker = bb.cooker.BBCooker(featureset, server)
+            cooker.configuration.profile = profile
         except bb.BBHandledException:
             return None
         writer.send("r")
@@ -549,7 +684,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         server.run()
     finally:
-        # Flush any ,essages/errors to the logfile before exit
+        # Flush any messages/errors to the logfile before exit
         sys.stdout.flush()
         sys.stderr.flush()
 
@@ -654,23 +789,18 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-        self.eventQueueLock.acquire()
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            if len(self.eventQueue) == 0:
+                return None
 
-        if len(self.eventQueue) == 0:
-            self.eventQueueNotify.clear()
+            item = self.eventQueue.pop(0)
+            if len(self.eventQueue) == 0:
+                self.eventQueueNotify.clear()
 
-        self.eventQueueLock.release()
         return item
 
     def waitEvent(self, delay):
@@ -678,10 +808,9 @@ class BBUIEventQueue:
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
@@ -690,13 +819,17 @@ class BBUIEventQueue:
         bb.utils.set_process_name("UIEventQueue")
         while True:
             try:
-                self.reader.wait()
-                event = self.reader.get()
-                self.queue_event(event)
-            except EOFError:
+                ready = self.reader.wait(0.25)
+                if ready:
+                    event = self.reader.get()
+                    self.queue_event(event)
+            except (EOFError, OSError, TypeError):
                 # Easiest way to exit is to close the file descriptor to cause an exit
                 break
+
+    def close(self):
         self.reader.close()
+        self.t.join()
 
 class ConnectionReader(object):
 
@@ -711,7 +844,7 @@ class ConnectionReader(object):
         return self.reader.poll(timeout)
 
     def get(self):
-        with self.rlock:
+        with bb.utils.lock_timeout(self.rlock):
             res = self.reader.recv_bytes()
         return multiprocessing.reduction.ForkingPickler.loads(res)
 
@@ -730,10 +863,31 @@ class ConnectionWriter(object):
         # Why bb.event needs this I have no idea
         self.event = self
 
+    def _send(self, obj):
+        gc.disable()
+        with bb.utils.lock_timeout(self.wlock):
+            self.writer.send_bytes(obj)
+        gc.enable()
+
     def send(self, obj):
         obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
-        with self.wlock:
-            self.writer.send_bytes(obj)
+        # See notes/code in CookerParser
+        # We must not terminate holding this lock else processes will hang.
+        # For SIGTERM, raising afterwards avoids this.
+        # For SIGINT, we don't want to have written partial data to the pipe.
+        # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
+        process = multiprocessing.current_process()
+        if process and hasattr(process, "queue_signals"):
+            with bb.utils.lock_timeout(process.signal_threadlock):
+                process.queue_signals = True
+                self._send(obj)
+                process.queue_signals = False
+
+                while len(process.signal_received) > 0:
+                    sig = process.signal_received.pop()
+                    process.handle_sig(sig, None)
+        else:
+            self._send(obj)
 
     def fileno(self):
         return self.writer.fileno()
diff --git a/bitbake/lib/bb/server/xmlrpcserver.py b/bitbake/lib/bb/server/xmlrpcserver.py
index 2fa71be667..04b0b17db1 100644
--- a/bitbake/lib/bb/server/xmlrpcserver.py
+++ b/bitbake/lib/bb/server/xmlrpcserver.py
@@ -11,6 +11,7 @@ import hashlib
 import time
 import inspect
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import bb.server.xmlrpcclient
 
 import bb
 
@@ -117,7 +118,7 @@ class BitBakeXMLRPCServerCommands():
         """
         Run a cooker command on the server
         """
-        return self.server.cooker.command.runCommand(command, self.server.readonly)
+        return self.server.cooker.command.runCommand(command, self.server.parent, self.server.readonly)
 
     def getEventHandle(self):
         return self.event_handle
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 0d88c6ec68..8ab08ec961 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -11,6 +13,10 @@ import pickle
 import bb.data
 import difflib
 import simplediff
+import json
+import types
+from contextlib import contextmanager
+import bb.compress.zstd
 from bb.checksum import FileChecksumCache
 from bb import runqueue
 import hashserv
@@ -19,6 +25,35 @@ import hashserv.client
 logger = logging.getLogger('BitBake.SigGen')
 hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 
+#find_siginfo and find_siginfo_version are set by the metadata siggen
+# The minimum version of the find_siginfo function we need
+find_siginfo_minversion = 2
+
+HASHSERV_ENVVARS = [
+    "SSL_CERT_DIR",
+    "SSL_CERT_FILE",
+    "NO_PROXY",
+    "HTTPS_PROXY",
+    "HTTP_PROXY"
+]
+
+def check_siggen_version(siggen):
+    if not hasattr(siggen, "find_siginfo_version"):
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
+    if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
+
+class SetEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set) or isinstance(obj, frozenset):
+            return dict(_set_object=list(sorted(obj)))
+        return json.JSONEncoder.default(self, obj)
+
+def SetDecoder(dct):
+    if '_set_object' in dct:
+        return frozenset(dct['_set_object'])
+    return dct
+
 def init(d):
     siggens = [obj for obj in globals().values()
                       if type(obj) is type and issubclass(obj, SignatureGenerator)]
@@ -27,7 +62,6 @@ def init(d):
     for sg in siggens:
         if desired == sg.name:
             return sg(d)
-            break
     else:
         logger.error("Invalid signature generator '%s', using default 'noop'\n"
                      "Available generators: %s", desired,
@@ -39,11 +73,6 @@ class SignatureGenerator(object):
     """
     name = "noop"
 
-    # If the derived class supports multiconfig datacaches, set this to True
-    # The default is False for backward compatibility with derived signature
-    # generators that do not understand multiconfig caches
-    supports_multiconfig_datacaches = False
-
     def __init__(self, data):
         self.basehash = {}
         self.taskhash = {}
@@ -61,9 +90,39 @@ class SignatureGenerator(object):
     def postparsing_clean_cache(self):
         return
 
+    def setup_datacache(self, datacaches):
+        self.datacaches = datacaches
+
+    def setup_datacache_from_datastore(self, mcfn, d):
+        # In task context we have no cache so setup internal data structures
+        # from the fully parsed data store provided
+
+        mc = d.getVar("__BBMULTICONFIG", False) or ""
+        tasks = d.getVar('__BBTASKS', False)
+
+        self.datacaches = {}
+        self.datacaches[mc] = types.SimpleNamespace()
+        setattr(self.datacaches[mc], "stamp", {})
+        self.datacaches[mc].stamp[mcfn] = d.getVar('STAMP')
+        setattr(self.datacaches[mc], "stamp_extrainfo", {})
+        self.datacaches[mc].stamp_extrainfo[mcfn] = {}
+        for t in tasks:
+            flag = d.getVarFlag(t, "stamp-extra-info")
+            if flag:
+                self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag
+
+    def get_cached_unihash(self, tid):
+        return None
+
     def get_unihash(self, tid):
+        unihash = self.get_cached_unihash(tid)
+        if unihash:
+            return unihash
         return self.taskhash[tid]
 
+    def get_unihashes(self, tids):
+        return {tid: self.get_unihash(tid) for tid in tids}
+
     def prep_taskhash(self, tid, deps, dataCaches):
         return
 
@@ -75,17 +134,51 @@ class SignatureGenerator(object):
         """Write/update the file checksum cache onto disk"""
         return
 
+    def stampfile_base(self, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        return self.datacaches[mc].stamp[mcfn]
+
+    def stampfile_mcfn(self, taskname, mcfn, extrainfo=True):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        if not stamp:
+            return
+
+        stamp_extrainfo = ""
+        if extrainfo:
+            taskflagname = taskname
+            if taskname.endswith("_setscene"):
+                taskflagname = taskname.replace("_setscene", "")
+            stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
+
+        return self.stampfile(stamp, mcfn, taskname, stamp_extrainfo)
+
     def stampfile(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
+    def stampcleanmask_mcfn(self, taskname, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        if not stamp:
+            return []
+
+        taskflagname = taskname
+        if taskname.endswith("_setscene"):
+            taskflagname = taskname.replace("_setscene", "")
+        stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
+
+        return self.stampcleanmask(stamp, mcfn, taskname, stamp_extrainfo)
+
     def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
-    def dump_sigtask(self, fn, task, stampbase, runtime):
+    def dump_sigtask(self, mcfn, task, stampbase, runtime):
         return
 
-    def invalidate_task(self, task, d, fn):
-        bb.build.del_stamp(task, d, fn)
+    def invalidate_task(self, task, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        bb.utils.remove(stamp)
 
     def dump_sigs(self, dataCache, options):
         return
@@ -108,40 +201,19 @@ class SignatureGenerator(object):
     def save_unitaskhashes(self):
         return
 
-    def set_setscene_tasks(self, setscene_tasks):
+    def copy_unitaskhashes(self, targetdir):
         return
 
-    @classmethod
-    def get_data_caches(cls, dataCaches, mc):
-        """
-        This function returns the datacaches that should be passed to signature
-        generator functions. If the signature generator supports multiconfig
-        caches, the entire dictionary of data caches is sent, otherwise a
-        special proxy is sent that support both index access to all
-        multiconfigs, and also direct access for the default multiconfig.
-
-        The proxy class allows code in this class itself to always use
-        multiconfig aware code (to ease maintenance), but derived classes that
-        are unaware of multiconfig data caches can still access the default
-        multiconfig as expected.
-
-        Do not override this function in derived classes; it will be removed in
-        the future when support for multiconfig data caches is mandatory
-        """
-        class DataCacheProxy(object):
-            def __init__(self):
-                pass
-
-            def __getitem__(self, key):
-                return dataCaches[key]
-
-            def __getattr__(self, name):
-                return getattr(dataCaches[mc], name)
+    def set_setscene_tasks(self, setscene_tasks):
+        return
 
-        if cls.supports_multiconfig_datacaches:
-            return dataCaches
+    def exit(self):
+        return
 
-        return DataCacheProxy()
+def build_pnid(mc, pn, taskname):
+    if mc:
+        return "mc:" + mc + ":" + pn + ":" + taskname
+    return pn + ":" + taskname
 
 class SignatureGeneratorBasic(SignatureGenerator):
     """
@@ -152,15 +224,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.basehash = {}
         self.taskhash = {}
         self.unihash = {}
-        self.taskdeps = {}
         self.runtaskdeps = {}
         self.file_checksum_values = {}
         self.taints = {}
-        self.gendeps = {}
-        self.lookupcache = {}
         self.setscenetasks = set()
-        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
-        self.taskwhitelist = None
+        self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split())
+        self.taskhash_ignore_tasks = None
         self.init_rundepcheck(data)
         checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
         if checksum_cache_file:
@@ -175,21 +244,21 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.tidtopn = {}
 
     def init_rundepcheck(self, data):
-        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
-        if self.taskwhitelist:
-            self.twl = re.compile(self.taskwhitelist)
+        self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None
+        if self.taskhash_ignore_tasks:
+            self.twl = re.compile(self.taskhash_ignore_tasks)
         else:
             self.twl = None
 
-    def _build_data(self, fn, d):
+    def _build_data(self, mcfn, d):
 
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars)
 
-        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
+        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, mcfn)
 
         for task in tasklist:
-            tid = fn + ":" + task
+            tid = mcfn + ":" + task
             if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]:
                 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid]))
                 bb.error("The following commands may help:")
@@ -200,11 +269,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("%s -Sprintdiff\n" % cmd)
             self.basehash[tid] = basehash[tid]
 
-        self.taskdeps[fn] = taskdeps
-        self.gendeps[fn] = gendeps
-        self.lookupcache[fn] = lookupcache
-
-        return taskdeps
+        return taskdeps, gendeps, lookupcache
 
     def set_setscene_tasks(self, setscene_tasks):
         self.setscenetasks = set(setscene_tasks)
@@ -212,35 +277,47 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def finalise(self, fn, d, variant):
 
         mc = d.getVar("__BBMULTICONFIG", False) or ""
+        mcfn = fn
         if variant or mc:
-            fn = bb.cache.realfn2virtual(fn, variant, mc)
+            mcfn = bb.cache.realfn2virtual(fn, variant, mc)
 
         try:
-            taskdeps = self._build_data(fn, d)
+            taskdeps, gendeps, lookupcache = self._build_data(mcfn, d)
         except bb.parse.SkipRecipe:
             raise
         except:
-            bb.warn("Error during finalise of %s" % fn)
+            bb.warn("Error during finalise of %s" % mcfn)
             raise
 
+        basehashes = {}
+        for task in taskdeps:
+            basehashes[task] = self.basehash[mcfn + ":" + task]
+
+        d.setVar("__siggen_basehashes", basehashes)
+        d.setVar("__siggen_gendeps", gendeps)
+        d.setVar("__siggen_varvals", lookupcache)
+        d.setVar("__siggen_taskdeps", taskdeps)
+
         #Slow but can be useful for debugging mismatched basehashes
-        #for task in self.taskdeps[fn]:
-        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
+        #self.setup_datacache_from_datastore(mcfn, d)
+        #for task in taskdeps:
+        #    self.dump_sigtask(mcfn, task, d.getVar("STAMP"), False)
 
-        for task in taskdeps:
-            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
+    def setup_datacache_from_datastore(self, mcfn, d):
+        super().setup_datacache_from_datastore(mcfn, d)
 
-    def postparsing_clean_cache(self):
-        #
-        # After parsing we can remove some things from memory to reduce our memory footprint
-        #
-        self.gendeps = {}
-        self.lookupcache = {}
-        self.taskdeps = {}
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        for attr in ["siggen_varvals", "siggen_taskdeps", "siggen_gendeps"]:
+            if not hasattr(self.datacaches[mc], attr):
+                setattr(self.datacaches[mc], attr, {})
+        self.datacaches[mc].siggen_varvals[mcfn] = d.getVar("__siggen_varvals")
+        self.datacaches[mc].siggen_taskdeps[mcfn] = d.getVar("__siggen_taskdeps")
+        self.datacaches[mc].siggen_gendeps[mcfn] = d.getVar("__siggen_gendeps")
 
     def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
         # Return True if we should keep the dependency, False to drop it
-        # We only manipulate the dependencies for packages not in the whitelist
+        # We only manipulate the dependencies for packages not in the ignore
+        # list
         if self.twl and not self.twl.search(recipename):
             # then process the actual dependencies
             if self.twl.search(depname):
@@ -258,38 +335,37 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
     def prep_taskhash(self, tid, deps, dataCaches):
 
-        (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
+        (mc, _, task, mcfn) = bb.runqueue.split_tid_mcfn(tid)
 
         self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
         self.runtaskdeps[tid] = []
         self.file_checksum_values[tid] = []
-        recipename = dataCaches[mc].pkg_fn[fn]
+        recipename = dataCaches[mc].pkg_fn[mcfn]
 
         self.tidtopn[tid] = recipename
+        # save hashfn for deps into siginfo?
+        for dep in deps:
+            (depmc, _, deptask, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
+            dep_pn = dataCaches[depmc].pkg_fn[depmcfn]
 
-        for dep in sorted(deps, key=clean_basepath):
-            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
-            depname = dataCaches[depmc].pkg_fn[depmcfn]
-            if not self.supports_multiconfig_datacaches and mc != depmc:
-                # If the signature generator doesn't understand multiconfig
-                # data caches, any dependency not in the same multiconfig must
-                # be skipped for backward compatibility
-                continue
-            if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
+            if not self.rundep_check(mcfn, recipename, task, dep, dep_pn, dataCaches):
                 continue
+
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
-            self.runtaskdeps[tid].append(dep)
 
-        if task in dataCaches[mc].file_checksums[fn]:
+            dep_pnid = build_pnid(depmc, dep_pn, deptask)
+            self.runtaskdeps[tid].append((dep_pnid, dep))
+
+        if task in dataCaches[mc].file_checksums[mcfn]:
             if self.checksum_cache:
-                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
             else:
-                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
             for (f,cs) in checksums:
                 self.file_checksum_values[tid].append((f,cs))
 
-        taskdep = dataCaches[mc].task_deps[fn]
+        taskdep = dataCaches[mc].task_deps[mcfn]
         if 'nostamp' in taskdep and task in taskdep['nostamp']:
             # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
             if tid in self.taints and self.taints[tid].startswith("nostamp:"):
@@ -300,7 +376,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 taint = str(uuid.uuid4())
                 self.taints[tid] = "nostamp:" + taint
 
-        taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
+        taint = self.read_taint(mcfn, task, dataCaches[mc].stamp[mcfn])
         if taint:
             self.taints[tid] = taint
             logger.warning("%s is tainted from a forced run" % tid)
@@ -310,22 +386,24 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def get_taskhash(self, tid, deps, dataCaches):
 
         data = self.basehash[tid]
-        for dep in self.runtaskdeps[tid]:
-            data = data + self.get_unihash(dep)
+        for dep in sorted(self.runtaskdeps[tid]):
+            data += self.get_unihash(dep[1])
 
-        for (f, cs) in self.file_checksum_values[tid]:
+        for (f, cs) in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
             if cs:
-                data = data + cs
+                if "/./" in f:
+                    data += "./" + f.split("/./")[1]
+                data += cs
 
         if tid in self.taints:
             if self.taints[tid].startswith("nostamp:"):
-                data = data + self.taints[tid][8:]
+                data += self.taints[tid][8:]
             else:
-                data = data + self.taints[tid]
+                data += self.taints[tid]
 
         h = hashlib.sha256(data.encode("utf-8")).hexdigest()
         self.taskhash[tid] = h
-        #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
+        #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task])
         return h
 
     def writeout_file_checksum_cache(self):
@@ -340,9 +418,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def save_unitaskhashes(self):
         self.unihash_cache.save(self.unitaskhashes)
 
-    def dump_sigtask(self, fn, task, stampbase, runtime):
+    def copy_unitaskhashes(self, targetdir):
+        self.unihash_cache.copyfile(targetdir)
 
-        tid = fn + ":" + task
+    def dump_sigtask(self, mcfn, task, stampbase, runtime):
+        tid = mcfn + ":" + task
+        mc = bb.runqueue.mc_from_tid(mcfn)
         referencestamp = stampbase
         if isinstance(runtime, str) and runtime.startswith("customfile"):
             sigfile = stampbase
@@ -357,29 +438,34 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         data = {}
         data['task'] = task
-        data['basewhitelist'] = self.basewhitelist
-        data['taskwhitelist'] = self.taskwhitelist
-        data['taskdeps'] = self.taskdeps[fn][task]
+        data['basehash_ignore_vars'] = self.basehash_ignore_vars
+        data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks
+        data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[mcfn][task]
         data['basehash'] = self.basehash[tid]
         data['gendeps'] = {}
         data['varvals'] = {}
-        data['varvals'][task] = self.lookupcache[fn][task]
-        for dep in self.taskdeps[fn][task]:
-            if dep in self.basewhitelist:
+        data['varvals'][task] = self.datacaches[mc].siggen_varvals[mcfn][task]
+        for dep in self.datacaches[mc].siggen_taskdeps[mcfn][task]:
+            if dep in self.basehash_ignore_vars:
                 continue
-            data['gendeps'][dep] = self.gendeps[fn][dep]
-            data['varvals'][dep] = self.lookupcache[fn][dep]
+            data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[mcfn][dep]
+            data['varvals'][dep] = self.datacaches[mc].siggen_varvals[mcfn][dep]
 
         if runtime and tid in self.taskhash:
-            data['runtaskdeps'] = self.runtaskdeps[tid]
-            data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]]
+            data['runtaskdeps'] = [dep[0] for dep in sorted(self.runtaskdeps[tid])]
+            data['file_checksum_values'] = []
+            for f,cs in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
+                if "/./" in f:
+                    data['file_checksum_values'].append(("./" + f.split("/./")[1], cs))
+                else:
+                    data['file_checksum_values'].append((os.path.basename(f), cs))
             data['runtaskhashes'] = {}
-            for dep in data['runtaskdeps']:
-                data['runtaskhashes'][dep] = self.get_unihash(dep)
+            for dep in self.runtaskdeps[tid]:
+                data['runtaskhashes'][dep[0]] = self.get_unihash(dep[1])
             data['taskhash'] = self.taskhash[tid]
             data['unihash'] = self.get_unihash(tid)
 
-        taint = self.read_taint(fn, task, referencestamp)
+        taint = self.read_taint(mcfn, task, referencestamp)
         if taint:
             data['taint'] = taint
 
@@ -396,13 +482,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
                 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
 
-        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
+        fd, tmpfile = bb.utils.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
         try:
-            with os.fdopen(fd, "wb") as stream:
-                p = pickle.dump(data, stream, -1)
-                stream.flush()
+            with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f:
+                json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder)
+                f.flush()
             os.chmod(tmpfile, 0o664)
-            os.rename(tmpfile, sigfile)
+            bb.utils.rename(tmpfile, sigfile)
         except (OSError, IOError) as err:
             try:
                 os.unlink(tmpfile)
@@ -410,18 +496,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 pass
             raise err
 
-    def dump_sigfn(self, fn, dataCaches, options):
-        if fn in self.taskdeps:
-            for task in self.taskdeps[fn]:
-                tid = fn + ":" + task
-                mc = bb.runqueue.mc_from_tid(tid)
-                if tid not in self.taskhash:
-                    continue
-                if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
-                    bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid)
-                    bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
-                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
-
 class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
     name = "basichash"
 
@@ -432,11 +506,11 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
         # If task is not in basehash, then error
         return self.basehash[tid]
 
-    def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
-        if taskname != "do_setscene" and taskname.endswith("_setscene"):
-            tid = fn + ":" + taskname[:-9]
+    def stampfile(self, stampbase, mcfn, taskname, extrainfo, clean=False):
+        if taskname.endswith("_setscene"):
+            tid = mcfn + ":" + taskname[:-9]
         else:
-            tid = fn + ":" + taskname
+            tid = mcfn + ":" + taskname
         if clean:
             h = "*"
         else:
@@ -444,29 +518,106 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
         return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
 
-    def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
-        return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
+    def stampcleanmask(self, stampbase, mcfn, taskname, extrainfo):
+        return self.stampfile(stampbase, mcfn, taskname, extrainfo, clean=True)
+
+    def invalidate_task(self, task, mcfn):
+        bb.note("Tainting hash to force rebuild of task %s, %s" % (mcfn, task))
 
-    def invalidate_task(self, task, d, fn):
-        bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
-        bb.build.write_taint(task, d, fn)
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+
+        taintfn = stamp + '.' + task + '.taint'
+
+        import uuid
+        bb.utils.mkdirhier(os.path.dirname(taintfn))
+        # The specific content of the taint file is not really important,
+        # we just need it to be random, so a random UUID is used
+        with open(taintfn, 'w') as taintf:
+            taintf.write(str(uuid.uuid4()))
 
 class SignatureGeneratorUniHashMixIn(object):
     def __init__(self, data):
         self.extramethod = {}
+        # NOTE: The cache only tracks hashes that exist. Hashes that don't
+        # exist are always queries from the server since it is possible for
+        # hashes to appear over time, but much less likely for them to
+        # disappear
+        self.unihash_exists_cache = set()
+        self.username = None
+        self.password = None
+        self.env = {}
+
+        origenv = data.getVar("BB_ORIGENV")
+        for e in HASHSERV_ENVVARS:
+            value = data.getVar(e)
+            if not value and origenv:
+                value = origenv.getVar(e)
+            if value:
+                self.env[e] = value
         super().__init__(data)
 
     def get_taskdata(self):
-        return (self.server, self.method, self.extramethod) + super().get_taskdata()
+        return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata()
 
     def set_taskdata(self, data):
-        self.server, self.method, self.extramethod = data[:3]
-        super().set_taskdata(data[3:])
+        self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7]
+        super().set_taskdata(data[7:])
+
+    def get_hashserv_creds(self):
+        if self.username and self.password:
+            return {
+                "username": self.username,
+                "password": self.password,
+            }
 
+        return {}
+
+    @contextmanager
+    def _client_env(self):
+        orig_env = os.environ.copy()
+        try:
+            for k, v in self.env.items():
+                os.environ[k] = v
+
+            yield
+        finally:
+            for k, v in self.env.items():
+                if k in orig_env:
+                    os.environ[k] = orig_env[k]
+                else:
+                    del os.environ[k]
+
+    @contextmanager
     def client(self):
-        if getattr(self, '_client', None) is None:
-            self._client = hashserv.create_client(self.server)
-        return self._client
+        with self._client_env():
+            if getattr(self, '_client', None) is None:
+                self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+            yield self._client
+
+    @contextmanager
+    def client_pool(self):
+        with self._client_env():
+            if getattr(self, '_client_pool', None) is None:
+                self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
+            yield self._client_pool
+
+    def reset(self, data):
+        self.__close_clients()
+        return super().reset(data)
+
+    def exit(self):
+        self.__close_clients()
+        return super().exit()
+
+    def __close_clients(self):
+        with self._client_env():
+            if getattr(self, '_client', None) is not None:
+                self._client.close()
+                self._client = None
+            if getattr(self, '_client_pool', None) is not None:
+                self._client_pool.close()
+                self._client_pool = None
 
     def get_stampfile_hash(self, tid):
         if tid in self.taskhash:
@@ -499,7 +650,7 @@ class SignatureGeneratorUniHashMixIn(object):
             return None
         return unihash
 
-    def get_unihash(self, tid):
+    def get_cached_unihash(self, tid):
         taskhash = self.taskhash[tid]
 
         # If its not a setscene task we can return
@@ -514,40 +665,105 @@ class SignatureGeneratorUniHashMixIn(object):
             self.unihash[tid] = unihash
             return unihash
 
-        # In the absence of being able to discover a unique hash from the
-        # server, make it be equivalent to the taskhash. The unique "hash" only
-        # really needs to be a unique string (not even necessarily a hash), but
-        # making it match the taskhash has a few advantages:
-        #
-        # 1) All of the sstate code that assumes hashes can be the same
-        # 2) It provides maximal compatibility with builders that don't use
-        #    an equivalency server
-        # 3) The value is easy for multiple independent builders to derive the
-        #    same unique hash from the same input. This means that if the
-        #    independent builders find the same taskhash, but it isn't reported
-        #    to the server, there is a better chance that they will agree on
-        #    the unique hash.
-        unihash = taskhash
+        return None
 
-        try:
-            method = self.method
-            if tid in self.extramethod:
-                method = method + self.extramethod[tid]
-            data = self.client().get_unihash(method, self.taskhash[tid])
-            if data:
-                unihash = data
+    def _get_method(self, tid):
+        method = self.method
+        if tid in self.extramethod:
+            method = method + self.extramethod[tid]
+
+        return method
+
+    def unihashes_exist(self, query):
+        if len(query) == 0:
+            return {}
+
+        uncached_query = {}
+        result = {}
+        for key, unihash in query.items():
+            if unihash in self.unihash_exists_cache:
+                result[key] = True
+            else:
+                uncached_query[key] = unihash
+
+        if self.max_parallel <= 1 or len(uncached_query) <= 1:
+            # No parallelism required. Make the query serially with the single client
+            with self.client() as client:
+                uncached_result = {
+                    key: client.unihash_exists(value) for key, value in uncached_query.items()
+                }
+        else:
+            with self.client_pool() as client_pool:
+                uncached_result = client_pool.unihashes_exist(uncached_query)
+
+        for key, exists in uncached_result.items():
+            if exists:
+                self.unihash_exists_cache.add(query[key])
+            result[key] = exists
+
+        return result
+
+    def get_unihash(self, tid):
+        return self.get_unihashes([tid])[tid]
+
+    def get_unihashes(self, tids):
+        """
+        For a iterable of tids, returns a dictionary that maps each tid to a
+        unihash
+        """
+        result = {}
+        queries = {}
+        query_result = {}
+
+        for tid in tids:
+            unihash = self.get_cached_unihash(tid)
+            if unihash:
+                result[tid] = unihash
+            else:
+                queries[tid] = (self._get_method(tid), self.taskhash[tid])
+
+        if len(queries) == 0:
+            return result
+
+        if self.max_parallel <= 1 or len(queries) <= 1:
+            # No parallelism required. Make the query serially with the single client
+            with self.client() as client:
+                for tid, args in queries.items():
+                    query_result[tid] = client.get_unihash(*args)
+        else:
+            with self.client_pool() as client_pool:
+                query_result = client_pool.get_unihashes(queries)
+
+        for tid, unihash in query_result.items():
+            # In the absence of being able to discover a unique hash from the
+            # server, make it be equivalent to the taskhash. The unique "hash" only
+            # really needs to be a unique string (not even necessarily a hash), but
+            # making it match the taskhash has a few advantages:
+            #
+            # 1) All of the sstate code that assumes hashes can be the same
+            # 2) It provides maximal compatibility with builders that don't use
+            #    an equivalency server
+            # 3) The value is easy for multiple independent builders to derive the
+            #    same unique hash from the same input. This means that if the
+            #    independent builders find the same taskhash, but it isn't reported
+            #    to the server, there is a better chance that they will agree on
+            #    the unique hash.
+            taskhash = self.taskhash[tid]
+            if unihash:
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
-                hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
+                hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
                 hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
-        except hashserv.client.HashConnectionError as e:
-            bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+                unihash = taskhash
 
-        self.set_unihash(tid, unihash)
-        self.unihash[tid] = unihash
-        return unihash
+
+            self.set_unihash(tid, unihash)
+            self.unihash[tid] = unihash
+            result[tid] = unihash
+
+        return result
 
     def report_unihash(self, path, task, d):
         import importlib
@@ -556,14 +772,14 @@ class SignatureGeneratorUniHashMixIn(object):
         unihash = d.getVar('BB_UNIHASH')
         report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
         tempdir = d.getVar('T')
-        fn = d.getVar('BB_FILENAME')
-        tid = fn + ':do_' + task
+        mcfn = d.getVar('BB_FILENAME')
+        tid = mcfn + ':do_' + task
         key = tid + ':' + taskhash
 
         if self.setscenetasks and tid not in self.setscenetasks:
             return
 
-        # This can happen if locked sigs are in action. Detect and just abort
+        # This can happen if locked sigs are in action. Detect and just exit
         if taskhash != self.taskhash[tid]:
             return
 
@@ -611,17 +827,19 @@ class SignatureGeneratorUniHashMixIn(object):
                 if tid in self.extramethod:
                     method = method + self.extramethod[tid]
 
-                data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
+                with self.client() as client:
+                    data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
+
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
                     hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
-                    bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
+                    bb.event.fire(bb.runqueue.taskUniHashUpdate(mcfn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
                     hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
-            except hashserv.client.HashConnectionError as e:
+            except ConnectionError as e:
                 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
         finally:
             if sigfile:
@@ -642,7 +860,9 @@ class SignatureGeneratorUniHashMixIn(object):
             if tid in self.extramethod:
                 method = method + self.extramethod[tid]
 
-            data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+            with self.client() as client:
+                data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+
             hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
 
             if data is None:
@@ -661,7 +881,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 # TODO: What to do here?
                 hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
 
-        except hashserv.client.HashConnectionError as e:
+        except ConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
         return False
@@ -675,20 +895,20 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
         super().init_rundepcheck(data)
         self.server = data.getVar('BB_HASHSERVE')
         self.method = "sstate_output_hash"
+        self.max_parallel = 1
 
-#
-# Dummy class used for bitbake-selftest
-#
-class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash):
-    name = "TestMulticonfigDepends"
-    supports_multiconfig_datacaches = True
+def clean_checksum_file_path(file_checksum_tuple):
+    f, cs = file_checksum_tuple
+    if "/./" in f:
+        return "./" + f.split("/./")[1]
+    return f
 
 def dump_this_task(outfile, d):
     import bb.parse
-    fn = d.getVar("BB_FILENAME")
+    mcfn = d.getVar("BB_FILENAME")
     task = "do_" + d.getVar("BB_CURRENTTASK")
-    referencestamp = bb.build.stamp_internal(task, d, None, True)
-    bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
+    referencestamp = bb.parse.siggen.stampfile_base(mcfn)
+    bb.parse.siggen.dump_sigtask(mcfn, task, outfile, "customfile:" + referencestamp)
 
 def init_colors(enable_color):
     """Initialise colour dict for passing to compare_sigfiles()"""
@@ -741,38 +961,15 @@ def list_inline_diff(oldlist, newlist, colors=None):
             ret.append(item)
     return '[%s]' % (', '.join(ret))
 
-def clean_basepath(basepath):
-    basepath, dir, recipe_task = basepath.rsplit("/", 2)
-    cleaned = dir + '/' + recipe_task
-
-    if basepath[0] == '/':
-        return cleaned
-
-    if basepath.startswith("mc:") and basepath.count(':') >= 2:
-        mc, mc_name, basepath = basepath.split(":", 2)
-        mc_suffix = ':mc:' + mc_name
-    else:
-        mc_suffix = ''
-
-    # mc stuff now removed from basepath. Whatever was next, if present will be the first
-    # suffix. ':/', recipe path start, marks the end of this. Something like
-    # 'virtual:a[:b[:c]]:/path...' (b and c being optional)
-    if basepath[0] != '/':
-        cleaned += ':' + basepath.split(':/', 1)[0]
-
-    return cleaned + mc_suffix
+# Handled renamed fields
+def handle_renames(data):
+    if 'basewhitelist' in data:
+        data['basehash_ignore_vars'] = data['basewhitelist']
+        del data['basewhitelist']
+    if 'taskwhitelist' in data:
+        data['taskhash_ignore_tasks'] = data['taskwhitelist']
+        del data['taskwhitelist']
 
-def clean_basepaths(a):
-    b = {}
-    for x in a:
-        b[clean_basepath(x)] = a[x]
-    return b
-
-def clean_basepaths_list(a):
-    b = []
-    for x in a:
-        b.append(clean_basepath(x))
-    return b
 
 def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     output = []
@@ -794,20 +991,29 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         formatparams.update(values)
         return formatstr.format(**formatparams)
 
-    with open(a, 'rb') as f:
-        p1 = pickle.Unpickler(f)
-        a_data = p1.load()
-    with open(b, 'rb') as f:
-        p2 = pickle.Unpickler(f)
-        b_data = p2.load()
-
-    def dict_diff(a, b, whitelist=set()):
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
+    try:
+        with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
+            b_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (b, str(err)))
+        raise err
+
+    for data in [a_data, b_data]:
+        handle_renames(data)
+
+    def dict_diff(a, b, ignored_vars=set()):
         sa = set(a.keys())
         sb = set(b.keys())
         common = sa & sb
         changed = set()
         for i in common:
-            if a[i] != b[i] and i not in whitelist:
+            if a[i] != b[i] and i not in ignored_vars:
                 changed.add(i)
         added = sb - sa
         removed = sa - sb
@@ -815,11 +1021,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
 
     def file_checksums_diff(a, b):
         from collections import Counter
-        # Handle old siginfo format
-        if isinstance(a, dict):
-            a = [(os.path.basename(f), cs) for f, cs in a.items()]
-        if isinstance(b, dict):
-            b = [(os.path.basename(f), cs) for f, cs in b.items()]
+
+        # Convert lists back to tuples
+        a = [(f[0], f[1]) for f in a]
+        b = [(f[0], f[1]) for f in b]
+
         # Compare lists, ensuring we can handle duplicate filenames if they exist
         removedcount = Counter(a)
         removedcount.subtract(b)
@@ -846,15 +1052,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         removed = [x[0] for x in removed]
         return changed, added, removed
 
-    if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
-        output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
-        if a_data['basewhitelist'] and b_data['basewhitelist']:
-            output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
+    if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']:
+        output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars']))
+        if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']:
+            output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars']))
 
-    if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
-        output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
-        if a_data['taskwhitelist'] and b_data['taskwhitelist']:
-            output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
+    if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']:
+        output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks']))
+        if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']:
+            output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks']))
 
     if a_data['taskdeps'] != b_data['taskdeps']:
         output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
@@ -862,23 +1068,23 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     if a_data['basehash'] != b_data['basehash'] and not collapsed:
         output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
 
-    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
+    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars'])
     if changed:
-        for dep in changed:
+        for dep in sorted(changed):
             output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
             if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
                 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
     if added:
-        for dep in added:
+        for dep in sorted(added):
             output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
     if removed:
-        for dep in removed:
+        for dep in sorted(removed):
             output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
 
 
     changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
     if changed:
-        for dep in changed:
+        for dep in sorted(changed):
             oldval = a_data['varvals'][dep]
             newval = b_data['varvals'][dep]
             if newval and oldval and ('\n' in oldval or '\n' in newval):
@@ -902,9 +1108,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
 
     if not 'file_checksum_values' in a_data:
-         a_data['file_checksum_values'] = {}
+         a_data['file_checksum_values'] = []
     if not 'file_checksum_values' in b_data:
-         b_data['file_checksum_values'] = {}
+         b_data['file_checksum_values'] = []
 
     changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed:
@@ -931,11 +1137,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 a = a_data['runtaskdeps'][idx]
                 b = b_data['runtaskdeps'][idx]
                 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
-                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
+                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
 
         if changed:
-            clean_a = clean_basepaths_list(a_data['runtaskdeps'])
-            clean_b = clean_basepaths_list(b_data['runtaskdeps'])
+            clean_a = a_data['runtaskdeps']
+            clean_b = b_data['runtaskdeps']
             if clean_a != clean_b:
                 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
             else:
@@ -948,7 +1154,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         b = b_data['runtaskhashes']
         changed, added, removed = dict_diff(a, b)
         if added:
-            for dep in added:
+            for dep in sorted(added):
                 bdep_found = False
                 if removed:
                     for bdep in removed:
@@ -956,9 +1162,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
                             bdep_found = True
                 if not bdep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (dep, b[dep]))
         if removed:
-            for dep in removed:
+            for dep in sorted(removed):
                 adep_found = False
                 if added:
                     for adep in added:
@@ -966,11 +1172,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
                             adep_found = True
                 if not adep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (dep, a[dep]))
         if changed:
-            for dep in changed:
+            for dep in sorted(changed):
                 if not collapsed:
-                    output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
+                    output.append(color_format("{color_title}Hash for task dependency %s changed{color_default} from %s to %s") % (dep, a[dep], b[dep]))
                 if callable(recursecb):
                     recout = recursecb(dep, a[dep], b[dep])
                     if recout:
@@ -980,6 +1186,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             # If a dependent hash changed, might as well print the line above and then defer to the changes in
                             # that hash since in all likelyhood, they're the same changes this task also saw.
                             output = [output[-1]] + recout
+                            break
 
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
@@ -1001,7 +1208,7 @@ def calc_basehash(sigdata):
         basedata = ''
 
     alldeps = sigdata['taskdeps']
-    for dep in alldeps:
+    for dep in sorted(alldeps):
         basedata = basedata + dep
         val = sigdata['varvals'][dep]
         if val is not None:
@@ -1017,6 +1224,8 @@ def calc_taskhash(sigdata):
 
     for c in sigdata['file_checksum_values']:
         if c[1]:
+            if "./" in c[0]:
+                data = data + c[0]
             data = data + c[1]
 
     if 'taint' in sigdata:
@@ -1031,32 +1240,37 @@ def calc_taskhash(sigdata):
 def dump_sigfile(a):
     output = []
 
-    with open(a, 'rb') as f:
-        p1 = pickle.Unpickler(f)
-        a_data = p1.load()
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
+
+    handle_renames(a_data)
 
-    output.append("basewhitelist: %s" % (a_data['basewhitelist']))
+    output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars'])))
 
-    output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
+    output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or [])))
 
     output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
 
     output.append("basehash: %s" % (a_data['basehash']))
 
-    for dep in a_data['gendeps']:
-        output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
+    for dep in sorted(a_data['gendeps']):
+        output.append("List of dependencies for variable %s is %s" % (dep, sorted(a_data['gendeps'][dep])))
 
-    for dep in a_data['varvals']:
+    for dep in sorted(a_data['varvals']):
         output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
 
     if 'runtaskdeps' in a_data:
-        output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
+        output.append("Tasks this task depends on: %s" % (sorted(a_data['runtaskdeps'])))
 
     if 'file_checksum_values' in a_data:
-        output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
+        output.append("This task depends on the checksums of files: %s" % (sorted(a_data['file_checksum_values'])))
 
     if 'runtaskhashes' in a_data:
-        for dep in a_data['runtaskhashes']:
+        for dep in sorted(a_data['runtaskhashes']):
             output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
 
     if 'taint' in a_data:
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 47bad6d1fa..66545a65af 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -39,7 +39,7 @@ class TaskData:
     """
     BitBake Task Data implementation
     """
-    def __init__(self, abort = True, skiplist = None, allowincomplete = False):
+    def __init__(self, halt = True, skiplist = None, allowincomplete = False):
         self.build_targets = {}
         self.run_targets = {}
 
@@ -57,7 +57,7 @@ class TaskData:
         self.failed_rdeps = []
         self.failed_fns = []
 
-        self.abort = abort
+        self.halt = halt
         self.allowincomplete = allowincomplete
 
         self.skiplist = skiplist
@@ -328,7 +328,7 @@ class TaskData:
         try:
             self.add_provider_internal(cfgData, dataCache, item)
         except bb.providers.NoProvider:
-            if self.abort:
+            if self.halt:
                 raise
             self.remove_buildtarget(item)
 
@@ -451,12 +451,12 @@ class TaskData:
         for target in self.build_targets:
             if fn in self.build_targets[target]:
                 self.build_targets[target].remove(fn)
-                if len(self.build_targets[target]) == 0:
+                if not self.build_targets[target]:
                     self.remove_buildtarget(target, missing_list)
         for target in self.run_targets:
             if fn in self.run_targets[target]:
                 self.run_targets[target].remove(fn)
-                if len(self.run_targets[target]) == 0:
+                if not self.run_targets[target]:
                     self.remove_runtarget(target, missing_list)
 
     def remove_buildtarget(self, target, missing_list=None):
@@ -479,7 +479,7 @@ class TaskData:
                     fn = tid.rsplit(":",1)[0]
                     self.fail_fn(fn, missing_list)
 
-        if self.abort and target in self.external_targets:
+        if self.halt and target in self.external_targets:
             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
             raise bb.providers.NoProvider(target)
 
@@ -516,7 +516,7 @@ class TaskData:
                     self.add_provider_internal(cfgData, dataCache, target)
                     added = added + 1
                 except bb.providers.NoProvider:
-                    if self.abort and target in self.external_targets and not self.allowincomplete:
+                    if self.halt and target in self.external_targets and not self.allowincomplete:
                         raise
                     if not self.allowincomplete:
                         self.remove_buildtarget(target)
diff --git a/bitbake/lib/bb/tests/codeparser.py b/bitbake/lib/bb/tests/codeparser.py
index 826a2d2f6d..f6585fb3aa 100644
--- a/bitbake/lib/bb/tests/codeparser.py
+++ b/bitbake/lib/bb/tests/codeparser.py
@@ -44,6 +44,7 @@ class VariableReferenceTest(ReferenceTest):
     def parseExpression(self, exp):
         parsedvar = self.d.expandWithRefs(exp, None)
         self.references = parsedvar.references
+        self.execs = parsedvar.execs
 
     def test_simple_reference(self):
         self.setEmptyVars(["FOO"])
@@ -61,6 +62,11 @@ class VariableReferenceTest(ReferenceTest):
         self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
+    def test_python_exec_reference(self):
+        self.parseExpression("${@eval('3 * 5')}")
+        self.assertReferences(set())
+        self.assertExecs(set(["eval"]))
+
 class ShellReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -111,9 +117,9 @@ ${D}${libdir}/pkgconfig/*.pc
         self.assertExecs(set(["sed"]))
 
     def test_parameter_expansion_modifiers(self):
-        # - and + are also valid modifiers for parameter expansion, but are
+        # -,+ and : are also valid modifiers for parameter expansion, but are
         # valid characters in bitbake variable names, so are not included here
-        for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
+        for i in ('=', '?', '#', '%', '##', '%%'):
             name = "foo%sbar" % i
             self.parseExpression("${%s}" % name)
             self.assertNotIn(name, self.references)
@@ -318,7 +324,7 @@ d.getVar(a(), False)
             "filename": "example.bb",
         })
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
 
@@ -365,7 +371,7 @@ esac
         self.d.setVarFlags("FOO", {"func": True})
         self.setEmptyVars(execs)
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["somevar", "inverted"] + execs))
 
@@ -375,7 +381,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -384,7 +390,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -399,7 +405,7 @@ esac
         # Check dependencies
         self.d.setVar('ANOTHERVAR', expr)
         self.d.setVar('TESTVAR', 'anothervalue testval testval2')
-        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
         self.assertEqual(sorted(values.splitlines()),
                          sorted([expr,
                           'TESTVAR{anothervalue} = Set',
@@ -412,11 +418,55 @@ esac
         # Check final value
         self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
 
+    def test_contains_vardeps_excluded(self):
+        # Check the ignored_vars option to build_dependencies is handled by contains functionality
+        varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
+        self.d.setVar('ANOTHERVAR', varval)
+        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
+        self.d.setVar('TESTVAR2', 'testval3')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
+        self.assertEqual(deps, set(["TESTVAR2"]))
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
+
+        # Check the vardepsexclude flag is handled by contains functionality
+        self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
+        self.assertEqual(deps, set(["TESTVAR2"]))
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
+
+    def test_contains_vardeps_override_operators(self):
+        # Check override operators handle dependencies correctly with the contains functionality
+        expr_plain = 'testval'
+        expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} '
+        expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}'
+        expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}'
+        # Check dependencies
+        self.d.setVar('ANOTHERVAR', expr_plain)
+        self.d.prependVar('ANOTHERVAR', expr_prepend)
+        self.d.appendVar('ANOTHERVAR', expr_append)
+        self.d.setVar('ANOTHERVAR:remove', expr_remove)
+        self.d.setVar('TESTVAR1', 'blah')
+        self.d.setVar('TESTVAR2', 'testval2')
+        self.d.setVar('TESTVAR3', 'no-testval')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()),
+                         sorted([
+                          expr_prepend + expr_plain + expr_append,
+                          '_remove of ' + expr_remove,
+                          'TESTVAR1{testval1} = Unset',
+                          'TESTVAR2{testval2} = Set',
+                          'TESTVAR3{no-testval} = Set',
+                          ]))
+        # Check final value
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2'])
+
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")
     #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
     #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
-    #    self.assertEquals(deps, set(["oe_libinstall"]))
+    #    self.assertEqual(deps, set(["oe_libinstall"]))
 
 
diff --git a/bitbake/lib/bb/tests/color.py b/bitbake/lib/bb/tests/color.py
index bf03750c69..bb70cb393d 100644
--- a/bitbake/lib/bb/tests/color.py
+++ b/bitbake/lib/bb/tests/color.py
@@ -20,7 +20,7 @@ class ProgressWatcher:
     def __init__(self):
         self._reports = []
 
-    def handle_event(self, event):
+    def handle_event(self, event, d):
         self._reports.append((event.progress, event.rate))
 
     def reports(self):
@@ -31,7 +31,7 @@ class ColorCodeTests(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
         self._progress_watcher = ProgressWatcher()
-        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event, data=self.d)
 
     def tearDown(self):
         bb.event.remove("bb.build.TaskProgress", None)
diff --git a/bitbake/lib/bb/tests/compression.py b/bitbake/lib/bb/tests/compression.py
new file mode 100644
index 0000000000..95af3f96d7
--- /dev/null
+++ b/bitbake/lib/bb/tests/compression.py
@@ -0,0 +1,100 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.compress.lz4
+import bb.compress.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+    def setUp(self):
+        self._t = tempfile.TemporaryDirectory()
+        self.tmpdir = Path(self._t.name)
+        self.addCleanup(self._t.cleanup)
+
+    def _file_helper(self, mode_suffix, data):
+        tmp_file = self.tmpdir / "compressed"
+
+        with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+            f.write(data)
+
+        with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+            read_data = f.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_file(self):
+        self._file_helper("t", "Hello")
+
+    def test_binary_file(self):
+        self._file_helper("b", "Hello".encode("utf-8"))
+
+    def _pipe_helper(self, mode_suffix, data):
+        rfd, wfd = os.pipe()
+        with open(rfd, "rb") as r, open(wfd, "wb") as w:
+            with self.do_open(r, mode="r" + mode_suffix) as decompress:
+                with self.do_open(w, mode="w" + mode_suffix) as compress:
+                    compress.write(data)
+                read_data = decompress.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_pipe(self):
+        self._pipe_helper("t", "Hello")
+
+    def test_binary_pipe(self):
+        self._pipe_helper("b", "Hello".encode("utf-8"))
+
+    def test_bad_decompress(self):
+        tmp_file = self.tmpdir / "compressed"
+        with tmp_file.open("wb") as f:
+            f.write(b"\x00")
+
+        with self.assertRaises(OSError):
+            with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+                data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("lz4c") is None:
+            self.skipTest("'lz4c' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.lz4.open(*args, **kwargs) as f:
+            yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("zstd") is None:
+            self.skipTest("'zstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, **kwargs) as f:
+            yield f
+
+
+class PZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("pzstd") is None:
+            self.skipTest("'pzstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, num_threads=2, **kwargs) as f:
+            yield f
diff --git a/bitbake/lib/bb/tests/cooker.py b/bitbake/lib/bb/tests/cooker.py
index c82d4b7b81..9e524ae345 100644
--- a/bitbake/lib/bb/tests/cooker.py
+++ b/bitbake/lib/bb/tests/cooker.py
@@ -1,6 +1,8 @@
 #
 # BitBake Tests for cooker.py
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/bb/tests/data.py b/bitbake/lib/bb/tests/data.py
index 1d4a64b109..cbc7c1ecd4 100644
--- a/bitbake/lib/bb/tests/data.py
+++ b/bitbake/lib/bb/tests/data.py
@@ -60,6 +60,15 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@5*12}")
         self.assertEqual(str(val), "60")
 
+    def test_python_snippet_w_dict(self):
+        val = self.d.expand("${@{ 'green': 1, 'blue': 2 }['green']}")
+        self.assertEqual(str(val), "1")
+
+    def test_python_unexpanded_multi(self):
+        self.d.setVar("bar", "${unsetvar}")
+        val = self.d.expand("${@2*2},${foo},${@d.getVar('foo') + ' ${bar}'},${foo}")
+        self.assertEqual(str(val), "4,value_of_foo,${@d.getVar('foo') + ' ${unsetvar}'},value_of_foo")
+
     def test_expand_in_python_snippet(self):
         val = self.d.expand("${@'boo ' + '${foo}'}")
         self.assertEqual(str(val), "boo value_of_foo")
@@ -68,6 +77,18 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
+    def test_python_snippet_function_reference(self):
+        self.d.setVar("TESTVAL", "testvalue")
+        self.d.setVar("testfunc", 'd.getVar("TESTVAL")')
+        context = bb.utils.get_context()
+        context["testfunc"] = lambda d: d.getVar("TESTVAL")
+        val = self.d.expand("${@testfunc(d)}")
+        self.assertEqual(str(val), "testvalue")
+
+    def test_python_snippet_builtin_metadata(self):
+        self.d.setVar("eval", "INVALID")
+        self.d.expand("${@eval('3')}")
+
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
@@ -245,35 +266,35 @@ class TestConcatOverride(unittest.TestCase):
 
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_prepend", "${FOO}:")
+        self.d.setVar("TEST:prepend", "${FOO}:")
         self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_prepend", "${FOO}:")
-        self.d.setVar("TEST_append", ":val2")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:prepend", "${FOO}:")
+        self.d.setVar("TEST:append", ":val2")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
     def test_append_unset(self):
-        self.d.setVar("TEST_prepend", "${FOO}:")
-        self.d.setVar("TEST_append", ":val2")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:prepend", "${FOO}:")
+        self.d.setVar("TEST:append", ":val2")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar")
 
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.assertEqual(self.d.getVar("TEST"), " bar")
 
     def test_remove_cleared(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "val bar")
 
@@ -281,42 +302,42 @@ class TestConcatOverride(unittest.TestCase):
     # (including that whitespace is preserved)
     def test_remove_inactive_override(self):
         self.d.setVar("TEST", "${VAL} ${BAR}    123")
-        self.d.setVar("TEST_remove_inactiveoverride", "val")
+        self.d.setVar("TEST:remove:inactiveoverride", "val")
         self.assertEqual(self.d.getVar("TEST"), "val bar    123")
 
     def test_doubleref_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
         self.assertEqual(self.d.getVar("TEST_TEST"), " bar  bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.assertEqual(self.d.getVar("TEST"), "")
 
     def test_remove_expansion(self):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
-        self.d.setVar("TEST_remove", "${BAR}/X")
+        self.d.setVar("TEST:remove", "${BAR}/X")
         self.assertEqual(self.d.getVar("TEST"), " Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
-        self.d.setVar("TEST_remove", "${BAR}")
+        self.d.setVar("TEST:remove", "${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "A  C ")
 
     def test_remove_preserve_whitespace(self):
         # When the removal isn't active, the original value should be preserved
         self.d.setVar("TEST", " A B")
-        self.d.setVar("TEST_remove", "C")
+        self.d.setVar("TEST:remove", "C")
         self.assertEqual(self.d.getVar("TEST"), " A B")
 
     def test_remove_preserve_whitespace2(self):
         # When the removal is active preserve the whitespace
         self.d.setVar("TEST", " A B")
-        self.d.setVar("TEST_remove", "B")
+        self.d.setVar("TEST:remove", "B")
         self.assertEqual(self.d.getVar("TEST"), " A ")
 
 class TestOverrides(unittest.TestCase):
@@ -329,81 +350,86 @@ class TestOverrides(unittest.TestCase):
         self.assertEqual(self.d.getVar("TEST"), "testvalue")
 
     def test_one_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
+        self.d.setVar("TEST:bar", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
     def test_one_override_unset(self):
-        self.d.setVar("TEST2_bar", "testvalue2")
+        self.d.setVar("TEST2:bar", "testvalue2")
 
         self.assertEqual(self.d.getVar("TEST2"), "testvalue2")
-        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
+        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2:bar'])
 
     def test_multiple_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_local", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:local", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
-        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
+        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST:foo', 'OVERRIDES', 'TEST:bar', 'TEST:local'])
 
     def test_multiple_combined_overrides(self):
-        self.d.setVar("TEST_local_foo_bar", "testvalue3")
+        self.d.setVar("TEST:local:foo:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_multiple_overrides_unset(self):
-        self.d.setVar("TEST2_local_foo_bar", "testvalue3")
+        self.d.setVar("TEST2:local:foo:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST2"), "testvalue3")
 
     def test_keyexpansion_override(self):
         self.d.setVar("LOCAL", "local")
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_${LOCAL}", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:${LOCAL}", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         bb.data.expandKeys(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_rename_override(self):
-        self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
+        self.d.setVar("ALTERNATIVE:ncurses-tools:class-target", "a")
         self.d.setVar("OVERRIDES", "class-target")
-        self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
-        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a")
+        self.d.renameVar("ALTERNATIVE:ncurses-tools", "ALTERNATIVE:lib32-ncurses-tools")
+        self.assertEqual(self.d.getVar("ALTERNATIVE:lib32-ncurses-tools"), "a")
 
     def test_underscore_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_some_val", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:some_val", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
+    # Test an override with _<numeric> in it based on a real world OE issue
+    def test_underscore_override_2(self):
+        self.d.setVar("TARGET_ARCH", "x86_64")
+        self.d.setVar("PN", "test-${TARGET_ARCH}")
+        self.d.setVar("VERSION", "1")
+        self.d.setVar("VERSION:pn-test-${TARGET_ARCH}", "2")
+        self.d.setVar("OVERRIDES", "pn-${PN}")
+        bb.data.expandKeys(self.d)
+        self.assertEqual(self.d.getVar("VERSION"), "2")
+
     def test_remove_with_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_some_val", "testvalue3 testvalue5")
-        self.d.setVar("TEST_some_val_remove", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:some_val", "testvalue3 testvalue5")
+        self.d.setVar("TEST:some_val:remove", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), " testvalue5")
 
     def test_append_and_override_1(self):
-        self.d.setVar("TEST_append", "testvalue2")
-        self.d.setVar("TEST_bar", "testvalue3")
+        self.d.setVar("TEST:append", "testvalue2")
+        self.d.setVar("TEST:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3testvalue2")
 
     def test_append_and_override_2(self):
-        self.d.setVar("TEST_append_bar", "testvalue2")
+        self.d.setVar("TEST:append:bar", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvaluetestvalue2")
 
     def test_append_and_override_3(self):
-        self.d.setVar("TEST_bar_append", "testvalue2")
+        self.d.setVar("TEST:bar:append", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
-    # Test an override with _<numeric> in it based on a real world OE issue
-    def test_underscore_override(self):
-        self.d.setVar("TARGET_ARCH", "x86_64")
-        self.d.setVar("PN", "test-${TARGET_ARCH}")
-        self.d.setVar("VERSION", "1")
-        self.d.setVar("VERSION_pn-test-${TARGET_ARCH}", "2")
-        self.d.setVar("OVERRIDES", "pn-${PN}")
-        bb.data.expandKeys(self.d)
-        self.assertEqual(self.d.getVar("VERSION"), "2")
+    def test_append_and_unused_override(self):
+        # Had a bug where an unused override append could return "" instead of None
+        self.d.setVar("BAR:append:unusedoverride", "testvalue2")
+        self.assertEqual(self.d.getVar("BAR"), None)
 
 class TestKeyExpansion(unittest.TestCase):
     def setUp(self):
@@ -498,7 +524,7 @@ class TaskHash(unittest.TestCase):
         d.setVar("VAR", "val")
         # Adding an inactive removal shouldn't change the hash
         d.setVar("BAR", "notbar")
-        d.setVar("MYCOMMAND_remove", "${BAR}")
+        d.setVar("MYCOMMAND:remove", "${BAR}")
         nexthash = gettask_bashhash("mytask", d)
         self.assertEqual(orighash, nexthash)
 
diff --git a/bitbake/lib/bb/tests/event.py b/bitbake/lib/bb/tests/event.py
index 9ca7e9bc8e..ef61891d30 100644
--- a/bitbake/lib/bb/tests/event.py
+++ b/bitbake/lib/bb/tests/event.py
@@ -13,6 +13,7 @@ import pickle
 import threading
 import time
 import unittest
+import tempfile
 from unittest.mock import Mock
 from unittest.mock import call
 
@@ -157,7 +158,7 @@ class EventHandlingTest(unittest.TestCase):
                                  self._test_process.event_handler,
                                  event,
                                  None)
-        self._test_process.event_handler.assert_called_once_with(event)
+        self._test_process.event_handler.assert_called_once_with(event, None)
 
     def test_fire_class_handlers(self):
         """ Test fire_class_handlers method """
@@ -175,10 +176,10 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
-        expected_event_handler1 = [call(event1)]
-        expected_event_handler2 = [call(event1),
-                                   call(event2),
-                                   call(event2)]
+        expected_event_handler1 = [call(event1, None)]
+        expected_event_handler2 = [call(event1, None),
+                                   call(event2, None),
+                                   call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -205,7 +206,7 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
         expected_event_handler1 = []
-        expected_event_handler2 = [call(event1)]
+        expected_event_handler2 = [call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -223,7 +224,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2)]
+        expected = [call(event1, None), call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -237,7 +238,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2), call(event1)]
+        expected = [call(event1, None), call(event2, None), call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -251,7 +252,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2), call(event1), call(event2)]
+        expected = [call(event1,None), call(event2, None), call(event1, None), call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -359,9 +360,10 @@ class EventHandlingTest(unittest.TestCase):
 
         event1 = bb.event.ConfigParsed()
         bb.event.fire(event1, None)
-        expected = [call(event1)]
+        expected = [call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
+        expected = [call(event1)]
         self.assertEqual(self._test_ui1.event.send.call_args_list,
                          expected)
 
@@ -450,10 +452,9 @@ class EventHandlingTest(unittest.TestCase):
             and disable threadlocks tests """
         bb.event.fire(bb.event.OperationStarted(), None)
 
-    def test_enable_threadlock(self):
+    def test_event_threadlock(self):
         """ Test enable_threadlock method """
         self._set_threadlock_test_mockups()
-        bb.event.enable_threadlock()
         self._set_and_run_threadlock_test_workers()
         # Calls to UI handlers should be in order as all the registered
         # handlers for the event coming from the first worker should be
@@ -461,20 +462,6 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._threadlock_test_calls,
                          ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"])
 
-
-    def test_disable_threadlock(self):
-        """ Test disable_threadlock method """
-        self._set_threadlock_test_mockups()
-        bb.event.disable_threadlock()
-        self._set_and_run_threadlock_test_workers()
-        # Calls to UI handlers should be intertwined together. Thanks to the
-        # delay in the registered handlers for the event coming from the first
-        # worker, the event coming from the second worker starts being
-        # processed before finishing handling the first worker event.
-        self.assertEqual(self._threadlock_test_calls,
-                         ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"])
-
-
 class EventClassesTest(unittest.TestCase):
     """ Event classes test class """
 
@@ -482,6 +469,8 @@ class EventClassesTest(unittest.TestCase):
 
     def setUp(self):
         bb.event.worker_pid = EventClassesTest._worker_pid
+        self.d = bb.data.init()
+        bb.parse.siggen = bb.siggen.init(self.d)
 
     def test_Event(self):
         """ Test the Event base class """
@@ -964,3 +953,24 @@ class EventClassesTest(unittest.TestCase):
         event = bb.event.FindSigInfoResult(result)
         self.assertEqual(event.result, result)
         self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_lineno_in_eventhandler(self):
+        # The error lineno is 5, not 4 since the first line is '\n'
+        error_line = """
+# Comment line1
+# Comment line2
+python test_lineno_in_eventhandler() {
+    This is an error line
+}
+addhandler test_lineno_in_eventhandler
+test_lineno_in_eventhandler[eventmask] = "bb.event.ConfigParsed"
+"""
+
+        with self.assertLogs() as logs:
+            f = tempfile.NamedTemporaryFile(suffix = '.bb')
+            f.write(bytes(error_line, "utf-8"))
+            f.flush()
+            d = bb.parse.handle(f.name, self.d)['']
+
+        output = "".join(logs.output)
+        self.assertTrue(" line 5\n" in output)
diff --git a/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html b/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html
new file mode 100644
index 0000000000..4a1eb4de13
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html
@@ -0,0 +1,59 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+  <title>Index of /debian/pool/main/m/minicom</title>
+ </head>
+ <body>
+<h1>Index of /debian/pool/main/m/minicom</h1>
+  <table>
+   <tr><th valign="top"><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th></tr>
+   <tr><th colspan="4"><hr></th></tr>
+<tr><td valign="top"><img src="/icons/back.gif" alt="[PARENTDIR]"></td><td><a href="/debian/pool/main/m/">Parent Directory</a></td><td>&nbsp;</td><td align="right">  - </td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.debian.tar.xz">minicom_2.7-1+deb8u1.debian.tar.xz</a></td><td align="right">2017-04-24 08:22  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.dsc">minicom_2.7-1+deb8u1.dsc</a></td><td align="right">2017-04-24 08:22  </td><td align="right">1.9K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_amd64.deb">minicom_2.7-1+deb8u1_amd64.deb</a></td><td align="right">2017-04-25 21:10  </td><td align="right">257K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armel.deb">minicom_2.7-1+deb8u1_armel.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">246K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armhf.deb">minicom_2.7-1+deb8u1_armhf.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">245K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_i386.deb">minicom_2.7-1+deb8u1_i386.deb</a></td><td align="right">2017-04-25 21:41  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.debian.tar.xz">minicom_2.7-1.1.debian.tar.xz</a></td><td align="right">2017-04-22 09:34  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.dsc">minicom_2.7-1.1.dsc</a></td><td align="right">2017-04-22 09:34  </td><td align="right">1.9K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_amd64.deb">minicom_2.7-1.1_amd64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_arm64.deb">minicom_2.7-1.1_arm64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">250K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armel.deb">minicom_2.7-1.1_armel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">255K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armhf.deb">minicom_2.7-1.1_armhf.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">254K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_i386.deb">minicom_2.7-1.1_i386.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">266K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips.deb">minicom_2.7-1.1_mips.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips64el.deb">minicom_2.7-1.1_mips64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mipsel.deb">minicom_2.7-1.1_mipsel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_ppc64el.deb">minicom_2.7-1.1_ppc64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_s390x.deb">minicom_2.7-1.1_s390x.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_amd64.deb">minicom_2.7.1-1+b1_amd64.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">262K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_arm64.deb">minicom_2.7.1-1+b1_arm64.deb</a></td><td align="right">2018-05-06 07:58  </td><td align="right">250K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armel.deb">minicom_2.7.1-1+b1_armel.deb</a></td><td align="right">2018-05-06 08:45  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armhf.deb">minicom_2.7.1-1+b1_armhf.deb</a></td><td align="right">2018-05-06 10:42  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_i386.deb">minicom_2.7.1-1+b1_i386.deb</a></td><td align="right">2018-05-06 08:55  </td><td align="right">266K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mips.deb">minicom_2.7.1-1+b1_mips.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mipsel.deb">minicom_2.7.1-1+b1_mipsel.deb</a></td><td align="right">2018-05-06 12:13  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_ppc64el.deb">minicom_2.7.1-1+b1_ppc64el.deb</a></td><td align="right">2018-05-06 09:10  </td><td align="right">260K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_s390x.deb">minicom_2.7.1-1+b1_s390x.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">257K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b2_mips64el.deb">minicom_2.7.1-1+b2_mips64el.deb</a></td><td align="right">2018-05-06 09:41  </td><td align="right">260K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.debian.tar.xz">minicom_2.7.1-1.debian.tar.xz</a></td><td align="right">2017-08-13 15:40  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.dsc">minicom_2.7.1-1.dsc</a></td><td align="right">2017-08-13 15:40  </td><td align="right">1.8K</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.1.orig.tar.gz">minicom_2.7.1.orig.tar.gz</a></td><td align="right">2017-08-13 15:40  </td><td align="right">855K</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.orig.tar.gz">minicom_2.7.orig.tar.gz</a></td><td align="right">2014-01-01 09:36  </td><td align="right">843K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.debian.tar.xz">minicom_2.8-2.debian.tar.xz</a></td><td align="right">2021-06-15 03:47  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.dsc">minicom_2.8-2.dsc</a></td><td align="right">2021-06-15 03:47  </td><td align="right">1.8K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_amd64.deb">minicom_2.8-2_amd64.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">280K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_arm64.deb">minicom_2.8-2_arm64.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">275K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armel.deb">minicom_2.8-2_armel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">271K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armhf.deb">minicom_2.8-2_armhf.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">272K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_i386.deb">minicom_2.8-2_i386.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">285K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mips64el.deb">minicom_2.8-2_mips64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">277K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mipsel.deb">minicom_2.8-2_mipsel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">278K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_ppc64el.deb">minicom_2.8-2_ppc64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">286K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_s390x.deb">minicom_2.8-2_s390x.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">275K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8.orig.tar.bz2">minicom_2.8.orig.tar.bz2</a></td><td align="right">2021-01-03 12:44  </td><td align="right">598K</td></tr>
+   <tr><th colspan="4"><hr></th></tr>
+</table>
+<address>Apache Server at ftp.debian.org Port 80</address>
+</body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html
new file mode 100644
index 0000000000..4e41af6d6a
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/2.10/</title>
+</head><body><h1>Index of /sources/libxml2/2.10/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="LATEST-IS-2.10.3" title="LATEST-IS-2.10.3">LATEST-IS-2.10.3</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.news" title="libxml2-2.10.0.news">libxml2-2.10.0.news</a></td><td class="size">7.1 KiB</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.sha256sum" title="libxml2-2.10.0.sha256sum">libxml2-2.10.0.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.tar.xz" title="libxml2-2.10.0.tar.xz">libxml2-2.10.0.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.news" title="libxml2-2.10.1.news">libxml2-2.10.1.news</a></td><td class="size">455 B</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.sha256sum" title="libxml2-2.10.1.sha256sum">libxml2-2.10.1.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.tar.xz" title="libxml2-2.10.1.tar.xz">libxml2-2.10.1.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.news" title="libxml2-2.10.2.news">libxml2-2.10.2.news</a></td><td class="size">309 B</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.sha256sum" title="libxml2-2.10.2.sha256sum">libxml2-2.10.2.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.tar.xz" title="libxml2-2.10.2.tar.xz">libxml2-2.10.2.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.news" title="libxml2-2.10.3.news">libxml2-2.10.3.news</a></td><td class="size">294 B</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.sha256sum" title="libxml2-2.10.3.sha256sum">libxml2-2.10.3.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.tar.xz" title="libxml2-2.10.3.tar.xz">libxml2-2.10.3.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html
new file mode 100644
index 0000000000..abdfdd0fa2
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html
@@ -0,0 +1,40 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/2.9/</title>
+</head><body><h1>Index of /sources/libxml2/2.9/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="LATEST-IS-2.9.14" title="LATEST-IS-2.9.14">LATEST-IS-2.9.14</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.0.sha256sum" title="libxml2-2.9.0.sha256sum">libxml2-2.9.0.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:27</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.0.tar.xz" title="libxml2-2.9.0.tar.xz">libxml2-2.9.0.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:27</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.1.sha256sum" title="libxml2-2.9.1.sha256sum">libxml2-2.9.1.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:28</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.1.tar.xz" title="libxml2-2.9.1.tar.xz">libxml2-2.9.1.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:28</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.10.sha256sum" title="libxml2-2.9.10.sha256sum">libxml2-2.9.10.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.10.tar.xz" title="libxml2-2.9.10.tar.xz">libxml2-2.9.10.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.11.sha256sum" title="libxml2-2.9.11.sha256sum">libxml2-2.9.11.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:43</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.11.tar.xz" title="libxml2-2.9.11.tar.xz">libxml2-2.9.11.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:43</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.12.sha256sum" title="libxml2-2.9.12.sha256sum">libxml2-2.9.12.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:45</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.12.tar.xz" title="libxml2-2.9.12.tar.xz">libxml2-2.9.12.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:45</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.news" title="libxml2-2.9.13.news">libxml2-2.9.13.news</a></td><td class="size">26.6 KiB</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.sha256sum" title="libxml2-2.9.13.sha256sum">libxml2-2.9.13.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.tar.xz" title="libxml2-2.9.13.tar.xz">libxml2-2.9.13.tar.xz</a></td><td class="size">3.1 MiB</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.news" title="libxml2-2.9.14.news">libxml2-2.9.14.news</a></td><td class="size">1.0 KiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.sha256sum" title="libxml2-2.9.14.sha256sum">libxml2-2.9.14.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.tar.xz" title="libxml2-2.9.14.tar.xz">libxml2-2.9.14.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.2.sha256sum" title="libxml2-2.9.2.sha256sum">libxml2-2.9.2.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:30</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.2.tar.xz" title="libxml2-2.9.2.tar.xz">libxml2-2.9.2.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:30</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.3.sha256sum" title="libxml2-2.9.3.sha256sum">libxml2-2.9.3.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:31</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.3.tar.xz" title="libxml2-2.9.3.tar.xz">libxml2-2.9.3.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:31</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.4.sha256sum" title="libxml2-2.9.4.sha256sum">libxml2-2.9.4.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.4.tar.xz" title="libxml2-2.9.4.tar.xz">libxml2-2.9.4.tar.xz</a></td><td class="size">2.9 MiB</td><td class="date">2022-Feb-14 18:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.5.sha256sum" title="libxml2-2.9.5.sha256sum">libxml2-2.9.5.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:35</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.5.tar.xz" title="libxml2-2.9.5.tar.xz">libxml2-2.9.5.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:35</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.6.sha256sum" title="libxml2-2.9.6.sha256sum">libxml2-2.9.6.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:36</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.6.tar.xz" title="libxml2-2.9.6.tar.xz">libxml2-2.9.6.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:36</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.7.sha256sum" title="libxml2-2.9.7.sha256sum">libxml2-2.9.7.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:37</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.7.tar.xz" title="libxml2-2.9.7.tar.xz">libxml2-2.9.7.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:37</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.8.sha256sum" title="libxml2-2.9.8.sha256sum">libxml2-2.9.8.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:39</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.8.tar.xz" title="libxml2-2.9.8.tar.xz">libxml2-2.9.8.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:39</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.9.sha256sum" title="libxml2-2.9.9.sha256sum">libxml2-2.9.9.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:40</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.9.tar.xz" title="libxml2-2.9.9.tar.xz">libxml2-2.9.9.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:40</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html
new file mode 100644
index 0000000000..c183e06a55
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html
@@ -0,0 +1,19 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/</title>
+</head><body><h1>Index of /sources/libxml2/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="2.0/" title="2.0">2.0/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.1/" title="2.1">2.1/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.10/" title="2.10">2.10/</a></td><td class="size">-</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="2.2/" title="2.2">2.2/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.3/" title="2.3">2.3/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.4/" title="2.4">2.4/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.5/" title="2.5">2.5/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.6/" title="2.6">2.6/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.7/" title="2.7">2.7/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:24</td></tr>
+<tr><td class="link"><a href="2.8/" title="2.8">2.8/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:26</td></tr>
+<tr><td class="link"><a href="2.9/" title="2.9">2.9/</a></td><td class="size">-</td><td class="date">2022-May-02 12:04</td></tr>
+<tr><td class="link"><a href="cache.json" title="cache.json">cache.json</a></td><td class="size">22.8 KiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 7b2dac7b86..85c1f79ff3 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -6,11 +6,14 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import contextlib
 import unittest
 import hashlib
 import tempfile
 import collections
 import os
+import signal
+import tarfile
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
@@ -18,9 +21,28 @@ from bb.tests.support.httpserver import HTTPService
 
 def skipIfNoNetwork():
     if os.environ.get("BB_SKIP_NETTESTS") == "yes":
-        return unittest.skip("Network tests being skipped")
+        return unittest.skip("network test")
     return lambda f: f
 
+class TestTimeout(Exception):
+    # Indicate to pytest that this is not a test suite
+    __test__ = False
+
+class Timeout():
+
+    def __init__(self, seconds):
+        self.seconds = seconds
+
+    def handle_timeout(self, signum, frame):
+        raise TestTimeout("Test failed: timeout reached")
+
+    def __enter__(self):
+        signal.signal(signal.SIGALRM, self.handle_timeout)
+        signal.alarm(self.seconds)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        signal.alarm(0)
+
 class URITest(unittest.TestCase):
     test_uris = {
         "http://www.google.com/index.html" : {
@@ -286,6 +308,21 @@ class URITest(unittest.TestCase):
             'params': {"someparam" : "1"},
             'query': {},
             'relative': True
+        },
+        "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": {
+            'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip',
+            'scheme': 'https',
+            'hostname': 'www.innodisk.com',
+            'port': None,
+            'hostport': 'www.innodisk.com',
+            'path': '/Download_file',
+            'userinfo': '',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {"downloadfilename" : "EGPL-T101.zip"},
+            'query': {"9BE0BF6657": None},
+            'relative': False
         }
 
     }
@@ -376,7 +413,7 @@ class FetcherTest(unittest.TestCase):
     def setUp(self):
         self.origdir = os.getcwd()
         self.d = bb.data.init()
-        self.tempdir = tempfile.mkdtemp()
+        self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
         self.dldir = os.path.join(self.tempdir, "download")
         os.mkdir(self.dldir)
         self.d.setVar("DL_DIR", self.dldir)
@@ -390,57 +427,94 @@ class FetcherTest(unittest.TestCase):
         if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
             print("Not cleaning up %s. Please remove manually." % self.tempdir)
         else:
+            bb.process.run('chmod u+rw -R %s' % self.tempdir)
             bb.utils.prunedir(self.tempdir)
 
+    def git(self, cmd, cwd=None):
+        if isinstance(cmd, str):
+            cmd = 'git -c safe.bareRepository=all ' + cmd
+        else:
+            cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd
+        if cwd is None:
+            cwd = self.gitdir
+        return bb.process.run(cmd, cwd=cwd)[0]
+
+    def git_init(self, cwd=None):
+        self.git('init', cwd=cwd)
+        # Explicitly set initial branch to master as
+        # a common setup is to use other default
+        # branch than master.
+        self.git(['checkout', '-b', 'master'], cwd=cwd)
+
+        try:
+            self.git(['config', 'user.email'], cwd=cwd)
+        except bb.process.ExecutionError:
+            self.git(['config', 'user.email', 'you@example.com'], cwd=cwd)
+
+        try:
+            self.git(['config', 'user.name'], cwd=cwd)
+        except bb.process.ExecutionError:
+            self.git(['config', 'user.name', 'Your Name'], cwd=cwd)
+
 class MirrorUriTest(FetcherTest):
 
     replaceuris = {
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
             : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 
-            : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
+            : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
         ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
             : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
-        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
             : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
-        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
             : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
-        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 
+        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
             : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
-        ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 
+        ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
             : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
         ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
             : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
         ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
             : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 
-            : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
+            : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
         ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
         ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
         ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
             : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
+            : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https")
+            : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master",
+        ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz",
+        ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH")
+            : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
+        ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz")
+            : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
 
         #Renaming files doesn't work
         #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
         #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
     }
 
-    mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
-                "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
-                "https://.*/.* file:///someotherpath/downloads/ \n" \
-                "http://.*/.* file:///someotherpath/downloads/ \n"
+    mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
+                "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
+                "https://.*/.* file:///someotherpath/downloads/ " \
+                "http://.*/.* file:///someotherpath/downloads/"
 
     def test_urireplace(self):
+        self.d.setVar("FILESPATH", ".")
         for k, v in self.replaceuris.items():
             ud = bb.fetch.FetchData(k[0], self.d)
             ud.setup_localpath(self.d)
@@ -463,8 +537,8 @@ class MirrorUriTest(FetcherTest):
 
     def test_mirror_of_mirror(self):
         # Test if mirror of a mirror works
-        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
-        mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
+        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
+        mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/"
         fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
         mirrors = bb.fetch2.mirror_from_string(mirrorvar)
         uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
@@ -473,8 +547,8 @@ class MirrorUriTest(FetcherTest):
                                 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
                                 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
 
-    recmirrorvar = "https://.*/[^/]*    http://AAAA/A/A/A/ \n" \
-                   "https://.*/[^/]*    https://BBBB/B/B/B/ \n"
+    recmirrorvar = "https://.*/[^/]*    http://AAAA/A/A/A/ " \
+                   "https://.*/[^/]*    https://BBBB/B/B/B/"
 
     def test_recursive(self):
         fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
@@ -488,15 +562,15 @@ class MirrorUriTest(FetcherTest):
 class GitDownloadDirectoryNamingTest(FetcherTest):
     def setUp(self):
         super(GitDownloadDirectoryNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_dir = "git.openembedded.org.bitbake"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_dir = "github.com.openembedded.bitbake.git"
 
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -536,16 +610,16 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
 class TarballNamingTest(FetcherTest):
     def setUp(self):
         super(TarballNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
 
         self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
@@ -570,9 +644,9 @@ class TarballNamingTest(FetcherTest):
 class GitShallowTarballNamingTest(FetcherTest):
     def setUp(self):
         super(GitShallowTarballNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
 
         self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -580,7 +654,7 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -602,6 +676,39 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.assertIn(self.mirror_tarball, dir)
 
 
+class CleanTarballTest(FetcherTest):
+    def setUp(self):
+        super(CleanTarballTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_contents_does_not_leak_info(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        mtime = bb.process.run('git log --all -1 --format=%ct',
+                cwd=os.path.join(self.unpackdir, 'git'))
+        self.assertEqual(len(mtime), 2)
+        mtime = int(mtime[0])
+
+        archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
+        self.assertNotEqual(len(archive.members), 0)
+        for member in archive.members:
+            if member.name == ".":
+                continue
+            self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name)
+            self.assertEqual(member.uid, 0, "uid for %s differs" % member.name)
+            self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name)
+            self.assertEqual(member.gid, 0, "gid for %s differs" % member.name)
+            self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
@@ -619,6 +726,9 @@ class FetcherLocalTest(FetcherTest):
         os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
         touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
         touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
+        bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir)
+        bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir)
+        bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir)
         self.d.setVar("FILESPATH", self.localsrcdir)
 
     def fetchUnpack(self, uris):
@@ -632,6 +742,11 @@ class FetcherLocalTest(FetcherTest):
         flst.sort()
         return flst
 
+    def test_local_checksum_fails_no_file(self):
+        self.d.setVar("SRC_URI", "file://404")
+        with self.assertRaises(bb.BBHandledException):
+            bb.fetch.get_checksum_file_list(self.d)
+
     def test_local(self):
         tree = self.fetchUnpack(['file://a', 'file://dir/c'])
         self.assertEqual(tree, ['a', 'dir/c'])
@@ -673,57 +788,58 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
-    def test_local_gitfetch_usehead(self):
+    def test_local_striplevel(self):
+        tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def test_local_striplevel_gzip(self):
+        tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def test_local_striplevel_bzip2(self):
+        tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def dummyGitTest(self, suffix):
         # Create dummy local Git repo
         src_dir = tempfile.mkdtemp(dir=self.tempdir,
                                    prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
+        self.gitdir = os.path.abspath(src_dir)
+        self.git_init()
+        self.git(['commit', '--allow-empty', '-m', 'Dummy commit'])
         # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
+        self.git(['checkout', '-b', 'my-devel'])
+        self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2'])
+        orig_rev = self.git(['rev-parse', 'HEAD']).strip()
 
         # Fetch and check revision
         self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1"
+        self.d.setVar("__BBSRCREV_SEEN", "1")
+        url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
+        unpack_rev = self.git(['rev-parse', 'HEAD'],
+                              cwd=os.path.join(self.unpackdir, 'git')).strip()
         self.assertEqual(orig_rev, unpack_rev)
 
+    def test_local_gitfetch_usehead(self):
+        self.dummyGitTest("usehead=1")
+
     def test_local_gitfetch_usehead_withname(self):
-        # Create dummy local Git repo
-        src_dir = tempfile.mkdtemp(dir=self.tempdir,
-                                   prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
-        # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
+        self.dummyGitTest("usehead=1;name=newName")
 
-        # Fetch and check revision
-        self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName"
-        fetcher = bb.fetch.Fetch([url], self.d)
-        fetcher.download()
-        fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
-        self.assertEqual(orig_rev, unpack_rev)
+    def test_local_gitfetch_shared(self):
+        self.dummyGitTest("usehead=1;name=sharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    def test_local_gitfetch_noshared(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        self.dummyGitTest("usehead=1;name=noSharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
 
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
@@ -831,12 +947,12 @@ class FetcherNoNetworkTest(FetcherTest):
 class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_fetch(self):
-        fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
         self.d.setVar("BB_NO_NETWORK", "1")
-        fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
         self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
@@ -844,21 +960,22 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_fetch_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
     def test_fetch_mirror_of_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
     def test_fetch_file_mirror_of_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("FILESPATH", ".")
+        self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         os.mkdir(self.dldir + "/some2where")
         fetcher.download()
@@ -866,16 +983,46 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_fetch_premirror(self):
-        self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
+    def test_fetch_specify_downloadfilename(self):
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    # BZ13039
+    def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
+        self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    def test_fetch_premirror_use_downloadfilename_to_fetch(self):
+        # Ensure downloadfilename is used when fetching from premirror.
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
     def gitfetcher(self, url1, url2):
         def checkrevision(self, fetcher):
             fetcher.unpack(self.unpackdir)
-            revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
+            revision = self.git(['rev-parse', 'HEAD'],
+                                cwd=os.path.join(self.unpackdir, 'git')).strip()
             self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
 
         self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
@@ -893,25 +1040,25 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_gitfetch(self):
-        url1 = url2 = "git://git.openembedded.org/bitbake"
+        url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.gitfetcher(url1, url2)
 
     @skipIfNoNetwork()
     def test_gitfetch_goodsrcrev(self):
         # SRCREV is set but matches rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
+        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
         self.gitfetcher(url1, url2)
 
     @skipIfNoNetwork()
     def test_gitfetch_badsrcrev(self):
         # SRCREV is set but does not match rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
+        url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
         self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
     @skipIfNoNetwork()
     def test_gitfetch_tagandrev(self):
         # SRCREV is set but does not match rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
+        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https"
         self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
     @skipIfNoNetwork()
@@ -920,7 +1067,7 @@ class FetcherNetworkTest(FetcherTest):
         # `usehead=1' and instead fetch the specified SRCREV. See
         # test_local_gitfetch_usehead() for a positive use of the usehead
         # feature.
-        url = "git://git.openembedded.org/bitbake;usehead=1"
+        url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
@@ -929,38 +1076,38 @@ class FetcherNetworkTest(FetcherTest):
         # `usehead=1' and instead fetch the specified SRCREV. See
         # test_local_gitfetch_usehead() for a positive use of the usehead
         # feature.
-        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName"
+        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
-        recipeurl = "git://git.openembedded.org/bitbake"
-        mirrorurl = "git://someserver.org/bitbake"
-        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+        mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
         self.gitfetcher(recipeurl, mirrorurl)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
-        recipeurl = "git://someserver.org/bitbake"
-        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
         self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
-        realurl = "git://git.openembedded.org/bitbake"
-        recipeurl = "git://someserver.org/bitbake"
+        realurl = "https://git.openembedded.org/bitbake"
+        recipeurl = "git://someserver.org/bitbake;protocol=https"
         self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
         os.chdir(self.tempdir)
-        bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
-        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
+        self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir)
+        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir))
         self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
     def test_git_submodule(self):
         # URL with ssh submodules
-        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
+        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https"
         # Original URL (comment this if you have ssh access to git.yoctoproject.org)
-        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
+        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -977,10 +1124,29 @@ class FetcherNetworkTest(FetcherTest):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
     @skipIfNoNetwork()
+    def test_git_submodule_restricted_network_premirrors(self):
+        # this test is to ensure that premirrors will be tried in restricted network
+        # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses
+        url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231"
+        # create a download directory to be used as premirror later
+        tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
+        dl_premirror = os.path.join(tempdir, "download-premirror")
+        os.mkdir(dl_premirror)
+        self.d.setVar("DL_DIR", dl_premirror)
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        # now use the premirror in restricted network
+        self.d.setVar("DL_DIR", self.dldir)
+        self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror)
+        self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain")
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+
+    @skipIfNoNetwork()
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
-        url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
+        url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -996,7 +1162,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1011,12 +1177,12 @@ class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
 
         # CLI11 that pulls in a newer nlohmann-json
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1030,7 +1196,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_aktualizr(self):
-        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1050,7 +1216,7 @@ class FetcherNetworkTest(FetcherTest):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 
         # This repository also has submodules where the module (name), path and url do not align
-        url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+        url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1073,6 +1239,15 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
         self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
 
+    @skipIfNoNetwork()
+    def test_git_submodule_reference_to_parent(self):
+        self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master"
+        self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1")
+        with Timeout(60):
+            fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+            with self.assertRaises(bb.fetch2.FetchError):
+                fetcher.download()
+
 class SVNTest(FetcherTest):
     def skipIfNoSvn():
         import shutil
@@ -1107,8 +1282,9 @@ class SVNTest(FetcherTest):
                        cwd=repo_dir)
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
-        # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
+        # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/)
+        # Use still accessible svn repo (only trunk to avoid longer downloads)
+        bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1136,8 +1312,8 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit")
 
     @skipIfNoSvn()
     def test_external_svn(self):
@@ -1150,49 +1326,49 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit")
 
 class TrustedNetworksTest(FetcherTest):
     def test_trusted_network(self):
         # Ensure trusted_network returns False when the host IS in the list.
-        url = "git://Someserver.org/foo;rev=1"
+        url = "git://Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_wild_trusted_network(self):
         # Ensure trusted_network returns true when the *.host IS in the list.
-        url = "git://Someserver.org/foo;rev=1"
+        url = "git://Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_prefix_wild_trusted_network(self):
         # Ensure trusted_network returns true when the prefix matches *.host.
-        url = "git://git.Someserver.org/foo;rev=1"
+        url = "git://git.Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_two_prefix_wild_trusted_network(self):
         # Ensure trusted_network returns true when the prefix matches *.host.
-        url = "git://something.git.Someserver.org/foo;rev=1"
+        url = "git://something.git.Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_port_trusted_network(self):
         # Ensure trusted_network returns True, even if the url specifies a port.
-        url = "git://someserver.org:8080/foo;rev=1"
+        url = "git://someserver.org:8080/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_untrusted_network(self):
         # Ensure trusted_network returns False when the host is NOT in the list.
-        url = "git://someserver.org/foo;rev=1"
+        url = "git://someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
         self.assertFalse(bb.fetch.trusted_network(self.d, url))
 
     def test_wild_untrusted_network(self):
         # Ensure trusted_network returns False when the host is NOT in the list.
-        url = "git://*.someserver.org/foo;rev=1"
+        url = "git://*.someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
         self.assertFalse(bb.fetch.trusted_network(self.d, url))
 
@@ -1202,14 +1378,17 @@ class URLHandle(unittest.TestCase):
        "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
        "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
        "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
-       "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
+       "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}),
        "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
+       "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}),
+       r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}),
     }
     # we require a pathname to encodeurl but users can still pass such urls to 
     # decodeurl and we need to handle them
     decodedata = datatable.copy()
     decodedata.update({
        "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
+       "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}),
     })
 
     def test_decodeurl(self):
@@ -1226,37 +1405,39 @@ class FetchLatestVersionTest(FetcherTest):
 
     test_git_uris = {
         # version pattern "X.Y.Z"
-        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "")
             : "1.99.4",
         # version pattern "vX.Y"
         # mirror of git.infradead.org since network issues interfered with testing
-        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
+        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "")
             : "1.5.0",
         # version pattern "pkg_name-X.Y"
         # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
-        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
+        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
-        ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
+        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
             : "1.2.0",
-        ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
+        ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
                 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
-        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
+        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "")
             : "0.4.3",
-        ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
+        ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "")
             : "11.0.0",
-        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "")
             : "1.3.59",
-        ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+        ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "")
             : "3.82+dbg0.9",
+        ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0")
+            : "0.28.0",
     }
 
     test_wget_uris = {
@@ -1272,13 +1453,16 @@ class FetchLatestVersionTest(FetcherTest):
         # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
         ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
             : "2.8.12.1",
+        # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz
+        ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "")
+            : "2.10.3",
         #
         # packages with versions only in current directory
         #
-        # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
+        # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
         ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
             : "2.19",
-        # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
+        # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
         ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
             : "20120814",
         #
@@ -1295,12 +1479,18 @@ class FetchLatestVersionTest(FetcherTest):
         #
         # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
         # https://github.com/apple/cups/releases
-        ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
+        ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
             : "2.0.0",
         # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
         # http://ftp.debian.org/debian/pool/main/d/db5.3/
-        ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
+        ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
             : "5.3.10",
+        #
+        # packages where the tarball compression changed in the new version
+        #
+        # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz
+        ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "")
+            : "2.8",
     }
 
     @skipIfNoNetwork()
@@ -1315,6 +1505,9 @@ class FetchLatestVersionTest(FetcherTest):
             self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
             r = bb.utils.vercmp_string(v, verstring)
             self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
+            if k[4]:
+                r = bb.utils.vercmp_string(verstring, k[4])
+                self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4]))
 
     def test_wget_latest_versionstring(self):
         testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
@@ -1341,17 +1534,14 @@ class FetchLatestVersionTest(FetcherTest):
 
 
 class FetchCheckStatusTest(FetcherTest):
-    test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
+    test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
                       "https://yoctoproject.org/",
-                      "https://yoctoproject.org/documentation",
-                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
+                      "https://docs.yoctoproject.org",
+                      "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
                       "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
-                      "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
-                      "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
-                      "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
                       # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
                       "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
                       ]
@@ -1389,7 +1579,7 @@ class GitMakeShallowTest(FetcherTest):
         FetcherTest.setUp(self)
         self.gitdir = os.path.join(self.tempdir, 'gitshallow')
         bb.utils.mkdirhier(self.gitdir)
-        bb.process.run('git init', cwd=self.gitdir)
+        self.git_init()
 
     def assertRefs(self, expected_refs):
         actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1403,13 +1593,6 @@ class GitMakeShallowTest(FetcherTest):
         actual_count = len(revs.splitlines())
         self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
 
-    def git(self, cmd):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        return bb.process.run(cmd, cwd=self.gitdir)[0]
-
     def make_shallow(self, args=None):
         if args is None:
             args = ['HEAD']
@@ -1512,13 +1695,13 @@ class GitShallowTest(FetcherTest):
         self.srcdir = os.path.join(self.tempdir, 'gitsource')
 
         bb.utils.mkdirhier(self.srcdir)
-        self.git('init', cwd=self.srcdir)
+        self.git_init(cwd=self.srcdir)
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
         self.d.delVar('MIRRORS')
 
-        uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
+        uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
         self.d.setVar('SRCREV', '${AUTOREV}')
         self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
@@ -1526,6 +1709,7 @@ class GitShallowTest(FetcherTest):
         self.d.setVar('BB_GIT_SHALLOW', '1')
         self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
         self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
 
     def assertRefs(self, expected_refs, cwd=None):
         if cwd is None:
@@ -1543,15 +1727,6 @@ class GitShallowTest(FetcherTest):
         actual_count = len(revs.splitlines())
         self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
 
-    def git(self, cmd, cwd=None):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        if cwd is None:
-            cwd = self.gitdir
-        return bb.process.run(cmd, cwd=cwd)[0]
-
     def add_empty_file(self, path, cwd=None, msg=None):
         if msg is None:
             msg = path
@@ -1594,6 +1769,7 @@ class GitShallowTest(FetcherTest):
 
         # fetch and unpack, from the shallow tarball
         bb.utils.remove(self.gitdir, recurse=True)
+        bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
         bb.utils.remove(ud.clonedir, recurse=True)
         bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
 
@@ -1745,7 +1921,7 @@ class GitShallowTest(FetcherTest):
 
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
-        self.git('init', cwd=smdir)
+        self.git_init(cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1753,11 +1929,11 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('bsub', cwd=smdir)
 
         self.git('submodule init', cwd=self.srcdir)
-        self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
+        self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
         self.git('submodule update', cwd=self.srcdir)
         self.git('commit -m submodule -a', cwd=self.srcdir)
 
-        uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
+        uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
         fetcher, ud = self.fetch_shallow(uri)
 
         # Verify the main repository is shallow
@@ -1775,7 +1951,7 @@ class GitShallowTest(FetcherTest):
 
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
-        self.git('init', cwd=smdir)
+        self.git_init(cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1783,7 +1959,7 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('bsub', cwd=smdir)
 
         self.git('submodule init', cwd=self.srcdir)
-        self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
+        self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
         self.git('submodule update', cwd=self.srcdir)
         self.git('commit -m submodule -a', cwd=self.srcdir)
 
@@ -1794,8 +1970,8 @@ class GitShallowTest(FetcherTest):
 
         # Set up the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
-        os.rename(self.dldir, mirrordir)
-        self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
+        bb.utils.rename(self.dldir, mirrordir)
+        self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir)
 
         # Fetch from the mirror
         bb.utils.remove(self.dldir, recurse=True)
@@ -1818,10 +1994,10 @@ class GitShallowTest(FetcherTest):
             self.git('annex init', cwd=self.srcdir)
             open(os.path.join(self.srcdir, 'c'), 'w').close()
             self.git('annex add c', cwd=self.srcdir)
-            self.git('commit -m annex-c -a', cwd=self.srcdir)
-            bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
+            self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
+            bb.process.run('chmod u+w -R %s' % self.srcdir)
 
-            uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
+            uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
             fetcher, ud = self.fetch_shallow(uri)
 
             self.assertRevCount(1)
@@ -1910,9 +2086,9 @@ class GitShallowTest(FetcherTest):
         # Set up the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
-        self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir)
 
-        os.rename(os.path.join(self.dldir, mirrortarball),
+        bb.utils.rename(os.path.join(self.dldir, mirrortarball),
                   os.path.join(mirrordir, mirrortarball))
 
         # Fetch from the mirror
@@ -2033,7 +2209,7 @@ class GitShallowTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_bitbake(self):
-        self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
+        self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
         self.git('config core.bare true', cwd=self.srcdir)
         self.git('fetch', cwd=self.srcdir)
 
@@ -2068,7 +2244,7 @@ class GitShallowTest(FetcherTest):
         self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
         self.d.setVar('BB_GIT_SHALLOW', '1')
         self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
-        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d)
         fetcher.download()
 
         bb.utils.remove(self.dldir + "/*.tar.gz")
@@ -2078,12 +2254,18 @@ class GitShallowTest(FetcherTest):
         self.assertIn("fstests.doap", dir)
 
 class GitLfsTest(FetcherTest):
+    def skipIfNoGitLFS():
+        import shutil
+        if not shutil.which('git-lfs'):
+            return unittest.skip('git-lfs not installed')
+        return lambda f: f
+
     def setUp(self):
         FetcherTest.setUp(self)
 
         self.gitdir = os.path.join(self.tempdir, 'git')
         self.srcdir = os.path.join(self.tempdir, 'gitsource')
-        
+
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
@@ -2091,22 +2273,18 @@ class GitLfsTest(FetcherTest):
 
         self.d.setVar('SRCREV', '${AUTOREV}')
         self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
 
         bb.utils.mkdirhier(self.srcdir)
-        self.git('init', cwd=self.srcdir)
-        with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
-            attrs.write('*.mp3 filter=lfs -text')
-        self.git(['add', '.gitattributes'], cwd=self.srcdir)
-        self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
+        self.git_init(cwd=self.srcdir)
+        self.commit_file('.gitattributes', '*.mp3 filter=lfs -text')
 
-    def git(self, cmd, cwd=None):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        if cwd is None:
-            cwd = self.gitdir
-        return bb.process.run(cmd, cwd=cwd)[0]
+    def commit_file(self, filename, content):
+        with open(os.path.join(self.srcdir, filename), "w") as f:
+            f.write(content)
+        self.git(["add", filename], cwd=self.srcdir)
+        self.git(["commit", "-m", "Change"], cwd=self.srcdir)
+        return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip()
 
     def fetch(self, uri=None, download=True):
         uris = self.d.getVar('SRC_URI').split()
@@ -2119,65 +2297,158 @@ class GitLfsTest(FetcherTest):
         ud = fetcher.ud[uri]
         return fetcher, ud
 
+    def get_real_git_lfs_file(self):
+        self.d.setVar('PATH', os.environ.get('PATH'))
+        fetcher, ud = self.fetch()
+        fetcher.unpack(self.d.getVar('WORKDIR'))
+        unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg")
+        return unpacked_lfs_file
+
+    @skipIfNoGitLFS()
+    def test_fetch_lfs_on_srcrev_change(self):
+        """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested"""
+        self.git(["lfs", "install", "--local"], cwd=self.srcdir)
+
+        @contextlib.contextmanager
+        def hide_upstream_repository():
+            """Hide the upstream repository to make sure that git lfs cannot pull from it"""
+            temp_name = self.srcdir + ".bak"
+            os.rename(self.srcdir, temp_name)
+            try:
+                yield
+            finally:
+                os.rename(temp_name, self.srcdir)
+
+        def fetch_and_verify(revision, filename, content):
+            self.d.setVar('SRCREV', revision)
+            fetcher, ud = self.fetch()
+
+            with hide_upstream_repository():
+                workdir = self.d.getVar('WORKDIR')
+                fetcher.unpack(workdir)
+
+                with open(os.path.join(workdir, "git", filename)) as f:
+                    self.assertEqual(f.read(), content)
+
+        commit_1 = self.commit_file("a.mp3", "version 1")
+        commit_2 = self.commit_file("a.mp3", "version 2")
+
+        self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
+
+        # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are
+        # available even when the upstream repository disappears.
+        fetch_and_verify(commit_2, "a.mp3", "version 2")
+        # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download
+        # folder.
+        fetch_and_verify(commit_1, "a.mp3", "version 1")
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_succeeds_without_lfs_param(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master")
+        f = self.get_real_git_lfs_file()
+        self.assertTrue(os.path.exists(f))
+        self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_succeeds(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1")
+        f = self.get_real_git_lfs_file()
+        self.assertTrue(os.path.exists(f))
+        self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_skips(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0")
+        f = self.get_real_git_lfs_file()
+        # This is the actual non-smudged placeholder file on the repo if git-lfs does not run
+        lfs_file = (
+                   'version https://git-lfs.github.com/spec/v1\n'
+                   'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n'
+                   'size 11423554\n'
+        )
+
+        with open(f) as fh:
+            self.assertEqual(lfs_file, fh.read())
+
+    @skipIfNoGitLFS()
     def test_lfs_enabled(self):
         import shutil
 
-        uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # Careful: suppress initial attempt at downloading until
-        # we know whether git-lfs is installed.
-        fetcher, ud = self.fetch(uri=None, download=False)
-        self.assertIsNotNone(ud.method._find_git_lfs)
-
-        # If git-lfs can be found, the unpack should be successful. Only
-        # attempt this with the real live copy of git-lfs installed.
-        if ud.method._find_git_lfs(self.d):
-            fetcher.download()
-            shutil.rmtree(self.gitdir, ignore_errors=True)
-            fetcher.unpack(self.d.getVar('WORKDIR'))
-
-        # If git-lfs cannot be found, the unpack should throw an error
-        with self.assertRaises(bb.fetch2.FetchError):
-            fetcher.download()
-            ud.method._find_git_lfs = lambda d: False
-            shutil.rmtree(self.gitdir, ignore_errors=True)
-            fetcher.unpack(self.d.getVar('WORKDIR'))
+        # With git-lfs installed, test that we can fetch and unpack
+        fetcher, ud = self.fetch()
+        shutil.rmtree(self.gitdir, ignore_errors=True)
+        fetcher.unpack(self.d.getVar('WORKDIR'))
 
+    @skipIfNoGitLFS()
     def test_lfs_disabled(self):
         import shutil
 
-        uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
+        uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # In contrast to test_lfs_enabled(), allow the implicit download
-        # done by self.fetch() to occur here. The point of this test case
-        # is to verify that the fetcher can survive even if the source
+        # Verify that the fetcher can survive even if the source
         # repository has Git LFS usage configured.
         fetcher, ud = self.fetch()
-        self.assertIsNotNone(ud.method._find_git_lfs)
-
-        # If git-lfs can be found, the unpack should be successful. A
-        # live copy of git-lfs is not required for this case, so
-        # unconditionally forge its presence.
-        ud.method._find_git_lfs = lambda d: True
-        shutil.rmtree(self.gitdir, ignore_errors=True)
         fetcher.unpack(self.d.getVar('WORKDIR'))
 
-        # If git-lfs cannot be found, the unpack should be successful
-        ud.method._find_git_lfs = lambda d: False
-        shutil.rmtree(self.gitdir, ignore_errors=True)
-        fetcher.unpack(self.d.getVar('WORKDIR'))
+    def test_lfs_enabled_not_installed(self):
+        import shutil
+
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
+
+        # Artificially assert that git-lfs is not installed, so
+        # we can verify a failure to unpack in it's absence.
+        old_find_git_lfs = ud.method._find_git_lfs
+        try:
+            # If git-lfs cannot be found, the unpack should throw an error
+            with self.assertRaises(bb.fetch2.FetchError):
+                fetcher.download()
+                ud.method._find_git_lfs = lambda d: False
+                shutil.rmtree(self.gitdir, ignore_errors=True)
+                fetcher.unpack(self.d.getVar('WORKDIR'))
+        finally:
+            ud.method._find_git_lfs = old_find_git_lfs
+
+    def test_lfs_disabled_not_installed(self):
+        import shutil
+
+        uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
+
+        # Artificially assert that git-lfs is not installed, so
+        # we can verify a failure to unpack in it's absence.
+        old_find_git_lfs = ud.method._find_git_lfs
+        try:
+            # Even if git-lfs cannot be found, the unpack should be successful
+            fetcher.download()
+            ud.method._find_git_lfs = lambda d: False
+            shutil.rmtree(self.gitdir, ignore_errors=True)
+            fetcher.unpack(self.d.getVar('WORKDIR'))
+        finally:
+            ud.method._find_git_lfs = old_find_git_lfs
 
 class GitURLWithSpacesTest(FetcherTest):
     test_git_urls = {
-        "git://tfs-example.org:22/tfs/example%20path/example.git" : {
-            'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
+        "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master',
             'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
             'path': '/tfs/example path/example.git'
         },
-        "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
-            'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
+        "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master',
             'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
             'path': '/tfs/example path/example repo.git'
         }
@@ -2201,11 +2472,129 @@ class GitURLWithSpacesTest(FetcherTest):
             self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
             self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
 
+class CrateTest(FetcherTest):
+    @skipIfNoNetwork()
+    def test_crate_url(self):
+
+        uri = "crate://crates.io/glob/0.2.11"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_matching_recipe(self):
+
+        self.d.setVar('BP', 'glob-0.2.11')
+
+        uri = "crate://crates.io/glob/0.2.11"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_params(self):
+
+        uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "aho-corasick-renamed")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_multi(self):
+
+        uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        ud = fetcher.ud[fetcher.urls[1]]
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "time-0.1.35")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_incorrect_cksum(self):
+        uri = "crate://crates.io/aho-corasick/0.7.20"
+        self.d.setVar('SRC_URI', uri)
+        self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest())
+
+        uris = self.d.getVar('SRC_URI').split()
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"):
+            fetcher.download()
+
 class NPMTest(FetcherTest):
     def skipIfNoNpm():
         import shutil
         if not shutil.which('npm'):
-            return unittest.skip('npm not installed, tests being skipped')
+            return unittest.skip('npm not installed')
         return lambda f: f
 
     @skipIfNoNpm()
@@ -2250,11 +2639,42 @@ class NPMTest(FetcherTest):
         ud = fetcher.ud[fetcher.urls[0]]
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
+
+        # Setup the mirror by renaming the download directory
+        mirrordir = os.path.join(self.tempdir, 'mirror')
+        bb.utils.rename(self.dldir, mirrordir)
+        os.mkdir(self.dldir)
+
+        # Configure the premirror to be used
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir)
+        self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
+
+        # Fetch again
+        self.assertFalse(os.path.exists(ud.localpath))
+        # The npm fetcher doesn't handle that the .resolved file disappears
+        # while the fetcher object exists, which it does when we rename the
+        # download directory to "mirror" above. Thus we need a new fetcher to go
+        # with the now empty download directory.
+        fetcher = bb.fetch.Fetch([url], self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
+
+    @skipIfNoNpm()
+    @skipIfNoNetwork()
+    def test_npm_premirrors_with_specified_filename(self):
+        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        # Fetch once to get a tarball
+        fetcher = bb.fetch.Fetch([url], self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
         # Setup the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
-        os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
+        os.replace(ud.localpath, mirrorfilename)
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename)
         self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
         # Fetch again
         self.assertFalse(os.path.exists(ud.localpath))
@@ -2274,7 +2694,7 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         # Update the resolved url to an invalid url
         with open(ud.resolvefile, 'r') as f:
             url = f.read()
@@ -2293,7 +2713,7 @@ class NPMTest(FetcherTest):
         url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
-        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz')))
         fetcher.unpack(self.unpackdir)
         unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
         self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
@@ -2323,7 +2743,7 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_registry_alternate(self):
-        url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
@@ -2433,6 +2853,45 @@ class NPMTest(FetcherTest):
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
+    def test_npmsw_git(self):
+        swfile = self.create_shrinkwrap_file({
+            'dependencies': {
+                'cookie': {
+                    'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
+                    'from': 'github:jshttp/cookie.git'
+                }
+            }
+        })
+        fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
+
+        swfile = self.create_shrinkwrap_file({
+            'dependencies': {
+                'cookie': {
+                    'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
+                    'from': 'jshttp/cookie.git'
+                }
+            }
+        })
+        fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
+
+        swfile = self.create_shrinkwrap_file({
+            'dependencies': {
+                'nodejs': {
+                    'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262',
+                    'from': 'gitlab:gitlab-examples/nodejs'
+                }
+            }
+        })
+        fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git')))
+
+    @skipIfNoNpm()
+    @skipIfNoNetwork()
     def test_npmsw_dev(self):
         swfile = self.create_shrinkwrap_file({
             'dependencies': {
@@ -2590,7 +3049,7 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
         # Fetch again
         self.assertFalse(os.path.exists(ud.localpath))
@@ -2619,7 +3078,7 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         # Fetch again with invalid url
         self.assertFalse(os.path.exists(ud.localpath))
         swfile = self.create_shrinkwrap_file({
@@ -2634,3 +3093,273 @@ class NPMTest(FetcherTest):
         fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
+
+class GitSharedTest(FetcherTest):
+    def setUp(self):
+        super(GitSharedTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
+
+    @skipIfNoNetwork()
+    def test_shared_unpack(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    @skipIfNoNetwork()
+    def test_noshared_unpack(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
+
+
+class FetchPremirroronlyLocalTest(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyLocalTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "bitbake"
+        self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+        self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname)
+        self.testfilename = "bitbake-fetch.test"
+
+    def make_git_repo(self):
+        recipeurl = "git:/git.fake.repo/bitbake"
+        os.makedirs(self.gitdir)
+        self.git_init(cwd=self.gitdir)
+        for i in range(0):
+            self.git_new_commit()
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+
+    def git_new_commit(self):
+        import random
+        os.unlink(os.path.join(self.mirrordir, self.mirrorname))
+        branch = self.git("branch --show-current", self.gitdir).split()
+        with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile:
+            testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random()))
+        self.git("add {}".format(self.testfilename), self.gitdir)
+        self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir)
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+        return self.git("rev-parse HEAD", self.gitdir).strip()
+
+    def git_new_branch(self, name):
+        self.git_new_commit()
+        head = self.git("rev-parse HEAD", self.gitdir).strip()
+        self.git("checkout -b {}".format(name), self.gitdir)
+        newrev = self.git_new_commit()
+        self.git("checkout {}".format(head), self.gitdir)
+        return newrev
+
+    def test_mirror_multiple_fetches(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher2.download()
+        fetcher2.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher3.download()
+        fetcher3.unpack(self.unpackdir)
+
+
+    def test_mirror_commit_nonexistent(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_mirror_commit_exists(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    def test_mirror_tarball_nonexistent(self):
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_mirror_tarball_multiple_branches(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        both branches have required revisions
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", branch2rev)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
+        fetcher.download()
+        fetcher.unpack(os.path.join(self.tempdir, "unpacked"))
+        unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename)
+        self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!")
+        with open(unpacked, 'r') as f:
+            content = f.read()
+            ## We expect to see testbranch1 in the file, not master, not testbranch2
+            self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!")
+
+    def test_mirror_tarball_multiple_branches_nobranch(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        Unbalanced name/branches raises ParameterError
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", branch2rev)
+        with self.assertRaises(bb.fetch2.ParameterError):
+            fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+    def test_mirror_tarball_multiple_branches_norev(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        one of the branches specifies non existing SRCREV
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+
+class FetchPremirroronlyNetworkTest(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyNetworkTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "fstests"
+        self.clonedir = os.path.join(self.tempdir, "git")
+        self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame))
+        self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "0")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+
+    def make_git_repo(self):
+        import shutil
+        self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz"
+        os.makedirs(self.clonedir)
+        self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir)
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+        shutil.rmtree(self.clonedir)
+
+    @skipIfNoNetwork()
+    def test_mirror_tarball_updated(self):
+        self.make_git_repo()
+        ## Upstream commit is in the mirror
+        self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+
+    @skipIfNoNetwork()
+    def test_mirror_tarball_outdated(self):
+        self.make_git_repo()
+        ## Upstream commit not in the mirror
+        self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+class FetchPremirroronlyMercurialTest(FetcherTest):
+    """ Test for premirrors with mercurial repos
+        the test covers also basic hg:// clone (see fetch_and_create_tarball
+    """
+    def skipIfNoHg():
+        import shutil
+        if not shutil.which('hg'):
+            return unittest.skip('Mercurial not installed')
+        return lambda f: f
+
+    def setUp(self):
+        super(FetchPremirroronlyMercurialTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "libgnt"
+        self.clonedir = os.path.join(self.tempdir, "hg")
+        self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt"
+        self.d.setVar("SRCREV", "53e8b422faaf")
+        self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz"
+
+    def fetch_and_create_tarball(self):
+        """
+        Ask bitbake to download repo and prepare mirror tarball for us
+        """
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname)
+        self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile))
+        ## moving tarball to mirror directory
+        os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname))
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0")
+
+
+    @skipIfNoNetwork()
+    @skipIfNoHg()
+    def test_premirror_mercurial(self):
+        self.fetch_and_create_tarball()
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+
+class FetchPremirroronlyBrokenTarball(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyBrokenTarball, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "bitbake"
+        self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
+        self.recipe_url = "git://git.fake.repo/bitbake;protocol=https"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+        with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz:
+            targz.write("This is not tar.gz file!")
+
+    def test_mirror_broken_download(self):
+        import sys
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs:
+            fetcher.download()
+        output = "".join(logs.output)
+        self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
diff --git a/bitbake/lib/bb/tests/parse.py b/bitbake/lib/bb/tests/parse.py
index 9e21e18425..72d1962e7e 100644
--- a/bitbake/lib/bb/tests/parse.py
+++ b/bitbake/lib/bb/tests/parse.py
@@ -98,8 +98,8 @@ exportD = "d"
 
 
     overridetest = """
-RRECOMMENDS_${PN} = "a"
-RRECOMMENDS_${PN}_libc = "b"
+RRECOMMENDS:${PN} = "a"
+RRECOMMENDS:${PN}:libc = "b"
 OVERRIDES = "libc:${PN}"
 PN = "gtk+"
 """
@@ -110,16 +110,16 @@ PN = "gtk+"
         self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         bb.data.expandKeys(d)
         self.assertEqual(d.getVar("RRECOMMENDS"), "b")
-        d.setVar("RRECOMMENDS_gtk+", "c")
+        d.setVar("RRECOMMENDS:gtk+", "c")
         self.assertEqual(d.getVar("RRECOMMENDS"), "c")
 
     overridetest2 = """
 EXTRA_OECONF = ""
-EXTRA_OECONF_class-target = "b"
-EXTRA_OECONF_append = " c"
+EXTRA_OECONF:class-target = "b"
+EXTRA_OECONF:append = " c"
 """
 
-    def test_parse_overrides(self):
+    def test_parse_overrides2(self):
         f = self.parsehelper(self.overridetest2)
         d = bb.parse.handle(f.name, self.d)['']
         d.appendVar("EXTRA_OECONF", " d")
@@ -128,7 +128,7 @@ EXTRA_OECONF_append = " c"
 
     overridetest3 = """
 DESCRIPTION = "A"
-DESCRIPTION_${PN}-dev = "${DESCRIPTION} B"
+DESCRIPTION:${PN}-dev = "${DESCRIPTION} B"
 PN = "bc"
 """
 
@@ -136,15 +136,15 @@ PN = "bc"
         f = self.parsehelper(self.overridetest3)
         d = bb.parse.handle(f.name, self.d)['']
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("DESCRIPTION_bc-dev"), "A B")
+        self.assertEqual(d.getVar("DESCRIPTION:bc-dev"), "A B")
         d.setVar("DESCRIPTION", "E")
-        d.setVar("DESCRIPTION_bc-dev", "C D")
+        d.setVar("DESCRIPTION:bc-dev", "C D")
         d.setVar("OVERRIDES", "bc-dev")
         self.assertEqual(d.getVar("DESCRIPTION"), "C D")
 
 
     classextend = """
-VAR_var_override1 = "B"
+VAR_var:override1 = "B"
 EXTRA = ":override1"
 OVERRIDES = "nothing${EXTRA}"
 
@@ -164,6 +164,7 @@ python () {
     # become unset/disappear.
     #
     def test_parse_classextend_contamination(self):
+        self.d.setVar("__bbclasstype", "recipe")
         cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass")
         #clsname = os.path.basename(cls.name).replace(".bbclass", "")
         self.classextend = self.classextend.replace("###CLASS###", cls.name)
@@ -185,12 +186,158 @@ deltask ${EMPTYVAR}
 """
     def test_parse_addtask_deltask(self):
         import sys
-        f = self.parsehelper(self.addtask_deltask)
+
+        with self.assertLogs() as logs:
+            f = self.parsehelper(self.addtask_deltask)
+            d = bb.parse.handle(f.name, self.d)['']
+
+        output = "".join(logs.output)
+        self.assertTrue("addtask contained multiple 'before' keywords" in output)
+        self.assertTrue("addtask contained multiple 'after' keywords" in output)
+        self.assertTrue('addtask ignored: " do_patch"' in output)
+        #self.assertTrue('dependent task do_foo for do_patch does not exist' in output)
+
+    broken_multiline_comment = """
+# First line of comment \\
+# Second line of comment \\
+
+"""
+    def test_parse_broken_multiline_comment(self):
+        f = self.parsehelper(self.broken_multiline_comment)
+        with self.assertRaises(bb.BBHandledException):
+            d = bb.parse.handle(f.name, self.d)['']
+
+
+    comment_in_var = """
+VAR = " \\
+    SOMEVAL \\
+#   some comment \\
+    SOMEOTHERVAL \\
+"
+"""
+    def test_parse_comment_in_var(self):
+        f = self.parsehelper(self.comment_in_var)
+        with self.assertRaises(bb.BBHandledException):
+            d = bb.parse.handle(f.name, self.d)['']
+
+
+    at_sign_in_var_flag = """
+A[flag@.service] = "nonet"
+B[flag@.target] = "ntb"
+C[f] = "flag"
+
+unset A[flag@.service]
+"""
+    def test_parse_at_sign_in_var_flag(self):
+        f = self.parsehelper(self.at_sign_in_var_flag)
         d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), None)
+        self.assertEqual(d.getVar("B"), None)
+        self.assertEqual(d.getVarFlag("A","flag@.service"), None)
+        self.assertEqual(d.getVarFlag("B","flag@.target"), "ntb")
+        self.assertEqual(d.getVarFlag("C","f"), "flag")
+
+    def test_parse_invalid_at_sign_in_var_flag(self):
+        invalid_at_sign = self.at_sign_in_var_flag.replace("B[f", "B[@f")
+        f = self.parsehelper(invalid_at_sign)
+        with self.assertRaises(bb.parse.ParseError):
+            d = bb.parse.handle(f.name, self.d)['']
+
+    export_function_recipe = """
+inherit someclass
+"""
+
+    export_function_recipe2 = """
+inherit someclass
+
+do_compile () {
+    false
+}
+
+python do_compilepython () {
+    bb.note("Something else")
+}
+
+"""
+    export_function_class = """
+someclass_do_compile() {
+    true
+}
+
+python someclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
+
+    export_function_class2 = """
+secondclass_do_compile() {
+    true
+}
+
+python secondclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
 
-        stdout = sys.stdout.getvalue()
-        self.assertTrue("addtask contained multiple 'before' keywords" in stdout)
-        self.assertTrue("addtask contained multiple 'after' keywords" in stdout)
-        self.assertTrue('addtask ignored: " do_patch"' in stdout)
-        #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout)
+    def test_parse_export_functions(self):
+        def check_function_flags(d):
+            self.assertEqual(d.getVarFlag("do_compile", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compilepython", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compile", "python"), None)
+            self.assertEqual(d.getVarFlag("do_compilepython", "python"), "1")
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            self.d.setVar("__bbclasstype", "recipe")
+            recipename = tempdir + "/recipe.bb"
+            os.makedirs(tempdir + "/classes")
+            with open(tempdir + "/classes/someclass.bbclass", "w") as f:
+                f.write(self.export_function_class)
+                f.flush()
+            with open(tempdir + "/classes/secondclass.bbclass", "w") as f:
+                f.write(self.export_function_class2)
+                f.flush()
+
+            with open(recipename, "w") as f:
+                f.write(self.export_function_recipe)
+                f.flush()
+            os.chdir(tempdir)
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            recipename2 = tempdir + "/recipe2.bb"
+            with open(recipename2, "w") as f:
+                f.write(self.export_function_recipe2)
+                f.flush()
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            with open(recipename, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+            with open(recipename2, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("secondclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("secondclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
 
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index efebf001a9..05d7fd07dd 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
 T = "${TMPDIR}/workdir/${PN}/temp"
 BB_NUMBER_THREADS = "4"
 
-BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE"
+BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK"
 
 include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index 3d51779d6c..cc87e8d6a8 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -29,13 +29,14 @@ class RunQueueTests(unittest.TestCase):
     def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
         env = os.environ.copy()
         env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
-        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
+        env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "SSTATEVALID SLOWTASKS TOPDIR"
         env["SSTATEVALID"] = sstatevalid
         env["SLOWTASKS"] = slowtasks
+        env["TOPDIR"] = builddir
         if extraenv:
             for k in extraenv:
                 env[k] = extraenv[k]
-                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
+                env["BB_ENV_PASSTHROUGH_ADDITIONS"] = env["BB_ENV_PASSTHROUGH_ADDITIONS"] + " " + k
         try:
             output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
             print(output)
@@ -58,6 +59,8 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -68,6 +71,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -77,6 +82,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -86,6 +93,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_all_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -95,6 +104,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_no_settasks(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1", "-c", "patch"]
@@ -103,6 +114,8 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_mix_covered_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"]
@@ -111,6 +124,7 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
 
     # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
     def test_mixed_direct_tasks_setscene_tasks(self):
@@ -122,6 +136,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     # This test slows down the execution of do_package_setscene until after other real tasks have
     # started running which tests for a bug where tasks were being lost from the buildable list of real
     # tasks if they weren't in tasks_covered or tasks_notcovered
@@ -136,12 +152,14 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
-    def test_setscenewhitelist(self):
+            self.shutdown(tempdir)
+
+    def test_setscene_ignore_tasks(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
             extraenv = {
                 "BB_SETSCENE_ENFORCE" : "1",
-                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
+                "BB_SETSCENE_ENFORCE_IGNORE_TASKS" : "a1:do_package_write_rpm a1:do_build"
             }
             sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
@@ -149,6 +167,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:package_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     # Tests for problems with dependencies between setscene tasks
     def test_no_setscenevalid_harddeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -162,6 +182,8 @@ class RunQueueTests(unittest.TestCase):
                         'd1:populate_sysroot', 'd1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_no_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -172,6 +194,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('a1:package_qa')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_a1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -182,6 +206,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_b1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -193,6 +219,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -203,6 +231,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_all_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -213,6 +243,8 @@ class RunQueueTests(unittest.TestCase):
                         'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_setscene_optimise(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -232,6 +264,8 @@ class RunQueueTests(unittest.TestCase):
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_bbmask(self):
         # This test validates that multiconfigs can independently mask off
         # recipes they do not want with BBMASK. It works by having recipes
@@ -248,11 +282,13 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_mcdepends(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
                 "BBMULTICONFIG" : "mc-1 mc_2",
-                "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
+                "BB_SIGNATURE_HANDLER" : "basichash",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
             tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
@@ -278,7 +314,8 @@ class RunQueueTests(unittest.TestCase):
                        ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
+            self.shutdown(tempdir)
+
     def test_hashserv_single(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -304,7 +341,6 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_double(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -329,7 +365,6 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_multiple_setscene(self):
         # Runs e1:do_package_setscene twice
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -361,7 +396,6 @@ class RunQueueTests(unittest.TestCase):
 
     def shutdown(self, tempdir):
         # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup
-        while os.path.exists(tempdir + "/hashserve.sock"):
+        while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal") or os.path.exists(tempdir + "/bitbake.lock")):
             time.sleep(0.5)
 
-
diff --git a/bitbake/lib/bb/tests/siggen.py b/bitbake/lib/bb/tests/siggen.py
index c21ab4e4fb..0dc67e6cc2 100644
--- a/bitbake/lib/bb/tests/siggen.py
+++ b/bitbake/lib/bb/tests/siggen.py
@@ -17,75 +17,12 @@ import bb.siggen
 
 class SiggenTest(unittest.TestCase):
 
-    def test_clean_basepath_simple_target_basepath(self):
-        basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask'
+    def test_build_pnid(self):
+        tests = {
+            ('', 'helloworld', 'do_sometask') : 'helloworld:do_sometask',
+            ('XX', 'helloworld', 'do_sometask') : 'mc:XX:helloworld:do_sometask',
+        }
 
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
+        for t in tests:
+            self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
 
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_basic_virtual_basepath(self):
-        basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_basepath(self):
-        basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_virtual_long_prefix_basepath(self):
-        basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_basepath(self):
-        basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_long_prefix_basepath(self):
-        basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-
-    # def test_clean_basepath_performance(self):
-    #     input_basepaths = [
-    #         'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #     ]
-
-    #     time_start = time.time()
-
-    #     i = 2000000
-    #     while i >= 0:
-    #         for basepath in input_basepaths:
-    #             bb.siggen.clean_basepath(basepath)
-    #         i -= 1
-
-    #     elapsed = time.time() - time_start
-    #     print('{} ({}s)'.format(self.id(), round(elapsed, 3)))
-
-    #     self.assertTrue(False)
diff --git a/bitbake/lib/bb/tests/utils.py b/bitbake/lib/bb/tests/utils.py
index a7ff33db52..c363f62d7d 100644
--- a/bitbake/lib/bb/tests/utils.py
+++ b/bitbake/lib/bb/tests/utils.py
@@ -418,7 +418,7 @@ MULTILINE = "  stuff \\
                                ['MULTILINE'],
                                handle_var)
 
-        testvalue = re.sub('\s+', ' ', value_in_callback.strip())
+        testvalue = re.sub(r'\s+', ' ', value_in_callback.strip())
         self.assertEqual(expected_value, testvalue)
 
 class EditBbLayersConf(unittest.TestCase):
@@ -666,3 +666,21 @@ class GetReferencedVars(unittest.TestCase):
 
         layers = [{"SRC_URI"}, {"QT_GIT", "QT_MODULE", "QT_MODULE_BRANCH_PARAM", "QT_GIT_PROTOCOL"}, {"QT_GIT_PROJECT", "QT_MODULE_BRANCH", "BPN"}, {"PN", "SPECIAL_PKGSUFFIX"}]
         self.check_referenced("${SRC_URI}", layers)
+
+
+class EnvironmentTests(unittest.TestCase):
+    def test_environment(self):
+        os.environ["A"] = "this is A"
+        self.assertIn("A", os.environ)
+        self.assertEqual(os.environ["A"], "this is A")
+        self.assertNotIn("B", os.environ)
+
+        with bb.utils.environment(B="this is B"):
+            self.assertIn("A", os.environ)
+            self.assertEqual(os.environ["A"], "this is A")
+            self.assertIn("B", os.environ)
+            self.assertEqual(os.environ["B"], "this is B")
+
+        self.assertIn("A", os.environ)
+        self.assertEqual(os.environ["A"], "this is A")
+        self.assertNotIn("B", os.environ)
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 763c329810..dcd3910cc4 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -10,6 +10,7 @@
 import logging
 import os
 import sys
+import time
 import atexit
 import re
 from collections import OrderedDict, defaultdict
@@ -52,6 +53,10 @@ class TinfoilDataStoreConnectorVarHistory:
     def remoteCommand(self, cmd, *args, **kwargs):
         return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
 
+    def emit(self, var, oval, val, o, d):
+        ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex)
+        o.write(ret)
+
     def __getattr__(self, name):
         if not hasattr(bb.data_smart.VariableHistory, name):
             raise AttributeError("VariableHistory has no such method %s" % name)
@@ -320,11 +325,11 @@ class Tinfoil:
         self.recipes_parsed = False
         self.quiet = 0
         self.oldhandlers = self.logger.handlers[:]
+        self.localhandlers = []
         if setup_logging:
             # This is the *client-side* logger, nothing to do with
             # logging messages from the server
             bb.msg.logger_create('BitBake', output)
-            self.localhandlers = []
             for handler in self.logger.handlers:
                 if handler not in self.oldhandlers:
                     self.localhandlers.append(handler)
@@ -440,11 +445,17 @@ class Tinfoil:
         to initialise Tinfoil and use it with config_only=True first and
         then conditionally call this function to parse recipes later.
         """
-        config_params = TinfoilConfigParameters(config_only=False)
+        config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
         self.run_actions(config_params)
         self.recipes_parsed = True
 
-    def run_command(self, command, *params):
+    def modified_files(self):
+        """
+        Notify the server it needs to revalidate it's caches since the client has modified files
+        """
+        self.run_command("revalidateCaches")
+
+    def run_command(self, command, *params, handle_events=True):
         """
         Run a command on the server (as implemented in bb.command).
         Note that there are two types of command - synchronous and
@@ -464,7 +475,7 @@ class Tinfoil:
         try:
             result = self.server_connection.connection.runCommand(commandline)
         finally:
-            while True:
+            while handle_events:
                 event = self.wait_event()
                 if not event:
                     break
@@ -489,7 +500,7 @@ class Tinfoil:
         Wait for an event from the server for the specified time.
         A timeout of 0 means don't wait if there are no events in the queue.
         Returns the next event in the queue or None if the timeout was
-        reached. Note that in order to recieve any events you will
+        reached. Note that in order to receive any events you will
         first need to set the internal event mask using set_event_mask()
         (otherwise whatever event mask the UI set up will be in effect).
         """
@@ -725,6 +736,7 @@ class Tinfoil:
 
         ret = self.run_command('buildTargets', targets, task)
         if handle_events:
+            lastevent = time.time()
             result = False
             # Borrowed from knotty, instead somewhat hackily we use the helper
             # as the object to store "shutdown" on
@@ -737,6 +749,7 @@ class Tinfoil:
                     try:
                         event = self.wait_event(0.25)
                         if event:
+                            lastevent = time.time()
                             if event_callback and event_callback(event):
                                 continue
                             if helper.eventHandler(event):
@@ -757,7 +770,7 @@ class Tinfoil:
                                 if parseprogress:
                                     parseprogress.update(event.progress)
                                 else:
-                                    bb.warn("Got ProcessProgress event for someting that never started?")
+                                    bb.warn("Got ProcessProgress event for something that never started?")
                                 continue
                             if isinstance(event, bb.event.ProcessFinished):
                                 if self.quiet > 1:
@@ -769,7 +782,7 @@ class Tinfoil:
                             if isinstance(event, bb.command.CommandCompleted):
                                 result = True
                                 break
-                            if isinstance(event, bb.command.CommandFailed):
+                            if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)):
                                 self.logger.error(str(event))
                                 result = False
                                 break
@@ -781,10 +794,13 @@ class Tinfoil:
                                 self.logger.error(str(event))
                                 result = False
                                 break
-
                         elif helper.shutdown > 1:
                             break
                         termfilter.updateFooter()
+                        if time.time() > (lastevent + (3*60)):
+                            if not self.run_command('ping', handle_events=False):
+                                print("\nUnable to ping server and no events, closing down...\n")
+                                return False
                     except KeyboardInterrupt:
                         termfilter.clearFooter()
                         if helper.shutdown == 1:
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 43aa592842..8b212b7803 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -45,7 +45,7 @@ from pprint import pformat
 import logging
 from datetime import datetime, timedelta
 
-from django.db import transaction, connection
+from django.db import transaction
 
 
 # pylint: disable=invalid-name
@@ -227,6 +227,12 @@ class ORMWrapper(object):
         build.completed_on = timezone.now()
         build.outcome = outcome
         build.save()
+
+        # We force a sync point here to force the outcome status commit,
+        # which resolves a race condition with the build completion takedown
+        transaction.set_autocommit(True)
+        transaction.set_autocommit(False)
+
         signal_runbuilds()
 
     def update_target_set_license_manifest(self, target, license_manifest_path):
@@ -483,14 +489,14 @@ class ORMWrapper(object):
 
             # we already created the root directory, so ignore any
             # entry for it
-            if len(path) == 0:
+            if not path:
                 continue
 
             parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1])
-            if len(parent_path) == 0:
+            if not parent_path:
                 parent_path = "/"
             parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
-            tf_obj = Target_File.objects.create(
+            Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -555,7 +561,7 @@ class ORMWrapper(object):
 
             parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
 
-            tf_obj = Target_File.objects.create(
+            Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -571,7 +577,7 @@ class ORMWrapper(object):
         assert isinstance(build_obj, Build)
         assert isinstance(target_obj, Target)
 
-        errormsg = ""
+        errormsg = []
         for p in packagedict:
             # Search name swtiches round the installed name vs package name
             # by default installed name == package name
@@ -633,10 +639,10 @@ class ORMWrapper(object):
                         packagefile_objects.append(Package_File( package = packagedict[p]['object'],
                             path = targetpath,
                             size = targetfilesize))
-                    if len(packagefile_objects):
+                    if packagefile_objects:
                         Package_File.objects.bulk_create(packagefile_objects)
                 except KeyError as e:
-                    errormsg += "  stpi: Key error, package %s key %s \n" % ( p, e )
+                    errormsg.append("  stpi: Key error, package %s key %s \n" % (p, e))
 
             # save disk installed size
             packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -673,13 +679,13 @@ class ORMWrapper(object):
                     logger.warning("Could not add dependency to the package %s "
                                    "because %s is an unknown package", p, px)
 
-        if len(packagedeps_objs) > 0:
+        if packagedeps_objs:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
         else:
             logger.info("No package dependencies created")
 
-        if len(errormsg) > 0:
-            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
+        if errormsg:
+            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", "".join(errormsg))
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
         Target_Image_File.objects.create(target=target_obj,
@@ -767,7 +773,7 @@ class ORMWrapper(object):
             packagefile_objects.append(Package_File( package = bp_object,
                                         path = path,
                                         size = package_info['FILES_INFO'][path] ))
-        if len(packagefile_objects):
+        if packagefile_objects:
             Package_File.objects.bulk_create(packagefile_objects)
 
         def _po_byname(p):
@@ -809,7 +815,7 @@ class ORMWrapper(object):
                 packagedeps_objs.append(Package_Dependency(  package = bp_object,
                     depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS))
 
-        if len(packagedeps_objs) > 0:
+        if packagedeps_objs:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
 
         return bp_object
@@ -826,7 +832,7 @@ class ORMWrapper(object):
                     desc = vardump[root_var]['doc']
             if desc is None:
                 desc = ''
-            if len(desc):
+            if desc:
                 HelpText.objects.get_or_create(build=build_obj,
                                                area=HelpText.VARIABLE,
                                                key=k, text=desc)
@@ -846,7 +852,7 @@ class ORMWrapper(object):
                                 file_name = vh['file'],
                                 line_number = vh['line'],
                                 operation = vh['op']))
-                if len(varhist_objects):
+                if varhist_objects:
                     VariableHistory.objects.bulk_create(varhist_objects)
 
 
@@ -893,9 +899,6 @@ class BuildInfoHelper(object):
         self.task_order = 0
         self.autocommit_step = 1
         self.server = server
-        # we use manual transactions if the database doesn't autocommit on us
-        if not connection.features.autocommits_when_autocommit_is_off:
-            transaction.set_autocommit(False)
         self.orm_wrapper = ORMWrapper()
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
@@ -1059,27 +1062,6 @@ class BuildInfoHelper(object):
 
         return recipe_info
 
-    def _get_path_information(self, task_object):
-        self._ensure_build()
-
-        assert isinstance(task_object, Task)
-        build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
-        build_stats_path = []
-
-        for t in self.internal_state['targets']:
-            buildname = self.internal_state['build'].build_name
-            pe, pv = task_object.recipe.version.split(":",1)
-            if len(pe) > 0:
-                package = task_object.recipe.name + "-" + pe + "_" + pv
-            else:
-                package = task_object.recipe.name + "-" + pv
-
-            build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
-                                                     buildname=buildname,
-                                                     package=package))
-
-        return build_stats_path
-
 
     ################################
     ## external available methods to store information
@@ -1313,12 +1295,11 @@ class BuildInfoHelper(object):
                 task_information['outcome'] = Task.OUTCOME_FAILED
                 del self.internal_state['taskdata'][identifier]
 
-        if not connection.features.autocommits_when_autocommit_is_off:
-            # we force a sync point here, to get the progress bar to show
-            if self.autocommit_step % 3 == 0:
-                transaction.set_autocommit(True)
-                transaction.set_autocommit(False)
-            self.autocommit_step += 1
+        # we force a sync point here, to get the progress bar to show
+        if self.autocommit_step % 3 == 0:
+            transaction.set_autocommit(True)
+            transaction.set_autocommit(False)
+        self.autocommit_step += 1
 
         self.orm_wrapper.get_update_task_object(task_information, True) # must exist
 
@@ -1404,7 +1385,7 @@ class BuildInfoHelper(object):
         assert 'pn' in event._depgraph
         assert 'tdepends' in event._depgraph
 
-        errormsg = ""
+        errormsg = []
 
         # save layer version priorities
         if 'layer-priorities' in event._depgraph.keys():
@@ -1496,7 +1477,7 @@ class BuildInfoHelper(object):
                 elif dep in self.internal_state['recipes']:
                     dependency = self.internal_state['recipes'][dep]
                 else:
-                    errormsg += "  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep)
+                    errormsg.append("  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep))
                     continue
                 recipe_dep = Recipe_Dependency(recipe=target,
                                                depends_on=dependency,
@@ -1537,8 +1518,8 @@ class BuildInfoHelper(object):
                 taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep ))
         Task_Dependency.objects.bulk_create(taskdeps_objects)
 
-        if len(errormsg) > 0:
-            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg)
+        if errormsg:
+            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", "".join(errormsg))
 
 
     def store_build_package_information(self, event):
@@ -1618,7 +1599,7 @@ class BuildInfoHelper(object):
 
         if 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
-            if len(self.internal_state['backlog']):
+            if self.internal_state['backlog']:
                 tempevent = self.internal_state['backlog'].pop()
                 logger.debug("buildinfohelper: Saving stored event %s "
                              % tempevent)
@@ -1765,7 +1746,6 @@ class BuildInfoHelper(object):
 
         buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
         machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
-        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
         # location of the manifest files for this build;
         # note that this file is only produced if an image is produced
@@ -1786,6 +1766,18 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
+        if len(image_targets) > 0:
+            #if there are image targets retrieve image_name
+            image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+            if not image_name:
+                #When build target is an image and image_name is not found as an environment variable
+                logger.info("IMAGE_NAME not found, extracting from bitbake command")
+                cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0]
+                #filter out tokens that are command line options
+                cmd = [token for token in cmd if not token.startswith('-')]
+                image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name
+                logger.info("IMAGE_NAME found as : %s " % image_name)
+
         for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the
@@ -1990,8 +1982,6 @@ class BuildInfoHelper(object):
             # Do not skip command line build events
             self.store_log_event(tempevent,False)
 
-        if not connection.features.autocommits_when_autocommit_is_off:
-            transaction.set_autocommit(True)
 
         # unset the brbe; this is to prevent subsequent command-line builds
         # being incorrectly attached to the previous Toaster-triggered build;
diff --git a/bitbake/lib/bb/ui/eventreplay.py b/bitbake/lib/bb/ui/eventreplay.py
new file mode 100644
index 0000000000..d62ecbfa56
--- /dev/null
+++ b/bitbake/lib/bb/ui/eventreplay.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file re-uses code spread throughout other Bitbake source files.
+# As such, all other copyrights belong to their own right holders.
+#
+
+
+import os
+import sys
+import json
+import pickle
+import codecs
+
+
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
+
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
+
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            decodedline = json.loads(line)
+            if 'allvariables' in decodedline:
+                self.variables = decodedline['allvariables']
+                return
+            if not 'vars' in decodedline:
+                raise ValueError
+            event_str = decodedline['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
+
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
+        """
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 0efa614dfc..f86999bb09 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -21,10 +21,11 @@ import fcntl
 import struct
 import copy
 import atexit
+from itertools import groupby
 
 from bb.ui import uihelper
 
-featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
+featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
@@ -178,7 +179,7 @@ class TerminalFilter(object):
             new[3] = new[3] & ~termios.ECHO
             termios.tcsetattr(fd, termios.TCSADRAIN, new)
             curses.setupterm()
-            if curses.tigetnum("colors") > 2:
+            if curses.tigetnum("colors") > 2 and os.environ.get('NO_COLOR', '') == '':
                 for h in handlers:
                     try:
                         h.formatter.enable_color()
@@ -227,7 +228,9 @@ class TerminalFilter(object):
 
     def keepAlive(self, t):
         if not self.cuu:
-            print("Bitbake still alive (%ds)" % t)
+            print("Bitbake still alive (no events for %ds). Active tasks:" % t)
+            for t in self.helper.running_tasks:
+                print(t)
             sys.stdout.flush()
 
     def updateFooter(self):
@@ -249,58 +252,68 @@ class TerminalFilter(object):
             return
         tasks = []
         for t in runningpids:
+            start_time = activetasks[t].get("starttime", None)
+            if start_time:
+                msg = "%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"])
+            else:
+                msg = "%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"])
             progress = activetasks[t].get("progress", None)
             if progress is not None:
                 pbar = activetasks[t].get("progressbar", None)
                 rate = activetasks[t].get("rate", None)
-                start_time = activetasks[t].get("starttime", None)
                 if not pbar or pbar.bouncing != (progress < 0):
                     if progress < 0:
-                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = True
                     else:
-                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = False
                     activetasks[t]["progressbar"] = pbar
-                tasks.append((pbar, progress, rate, start_time))
+                tasks.append((pbar, msg, progress, rate, start_time))
             else:
-                start_time = activetasks[t].get("starttime", None)
-                if start_time:
-                    tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"]))
-                else:
-                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]))
+                tasks.append(msg)
 
         if self.main.shutdown:
-            content = "Waiting for %s running tasks to finish:" % len(activetasks)
+            content = pluralise("Waiting for %s running task to finish",
+                                "Waiting for %s running tasks to finish", len(activetasks))
+            if not self.quiet:
+                content += ':'
             print(content)
         else:
+            scene_tasks = "%s of %s" % (self.helper.setscene_current, self.helper.setscene_total)
+            cur_tasks = "%s of %s" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+
+            content = ''
+            if not self.quiet:
+                msg = "Setscene tasks: %s" % scene_tasks
+                content += msg + "\n"
+                print(msg)
+
             if self.quiet:
-                content = "Running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "Running tasks (%s, %s)" % (scene_tasks, cur_tasks)
             elif not len(activetasks):
-                content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "No currently running tasks (%s)" % cur_tasks
             else:
-                content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "Currently %2s running tasks (%s)" % (len(activetasks), cur_tasks)
             maxtask = self.helper.tasknumber_total
             if not self.main_progress or self.main_progress.maxval != maxtask:
                 widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
                 self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets, resize_handler=self.sigwinch_handle)
                 self.main_progress.start(False)
-            self.main_progress.setmessage(content)
-            progress = self.helper.tasknumber_current - 1
-            if progress < 0:
-                progress = 0
-            content = self.main_progress.update(progress)
+            self.main_progress.setmessage(msg)
+            progress = max(0, self.helper.tasknumber_current - 1)
+            content += self.main_progress.update(progress)
             print('')
-        lines = 1 + int(len(content) / (self.columns + 1))
-        if self.quiet == 0:
-            for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
+        lines = self.getlines(content)
+        if not self.quiet:
+            for tasknum, task in enumerate(tasks[:(self.rows - 1 - lines)]):
                 if isinstance(task, tuple):
-                    pbar, progress, rate, start_time = task
+                    pbar, msg, progress, rate, start_time = task
                     if not pbar.start_time:
                         pbar.start(False)
                         if start_time:
                             pbar.start_time = start_time
-                    pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
+                    pbar.setmessage('%s: %s' % (tasknum, msg))
                     pbar.setextra(rate)
                     if progress > -1:
                         content = pbar.update(progress)
@@ -310,11 +323,17 @@ class TerminalFilter(object):
                 else:
                     content = "%s: %s" % (tasknum, task)
                     print(content)
-                lines = lines + 1 + int(len(content) / (self.columns + 1))
+                lines = lines + self.getlines(content)
         self.footer_present = lines
         self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
 
+    def getlines(self, content):
+        lines = 0
+        for line in content.split("\n"):
+            lines = lines + 1 + int(len(line) / (self.columns + 1))
+        return lines
+
     def finish(self):
         if self.stdinbackup:
             fd = sys.stdin.fileno()
@@ -401,6 +420,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     except bb.BBHandledException:
         drain_events_errorhandling(eventHandler)
         return 1
+    except Exception as e:
+        # bitbake-server comms failure
+        early_logger = bb.msg.logger_create('bitbake', sys.stdout)
+        early_logger.fatal("Attempting to set server environment: %s", e)
+        return 1
 
     if params.options.quiet == 0:
         console_loglevel = loglevel
@@ -539,6 +563,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         except OSError:
            pass
 
+    # Add the logging domains specified by the user on the command line
+    for (domainarg, iterator) in groupby(params.debug_domains):
+        dlevel = len(tuple(iterator))
+        l = logconfig["loggers"].setdefault("BitBake.%s" % domainarg, {})
+        l["level"] = logging.DEBUG - dlevel + 1
+        l.setdefault("handlers", []).extend(["BitBake.verbconsole"])
+
     conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
 
     if sys.stdin.isatty() and sys.stdout.isatty():
@@ -559,7 +590,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         return
 
     llevel, debug_domains = bb.msg.constructLogOptions()
-    server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    try:
+        server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    except (BrokenPipeError, EOFError) as e:
+        # bitbake-server comms failure
+        logger.fatal("Attempting to set event mask: %s", e)
+        return 1
 
     # The logging_tree module is *extremely* helpful in debugging logging
     # domains. Uncomment here to dump the logging tree when bitbake starts
@@ -568,7 +604,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     universe = False
     if not params.observe_only:
-        params.updateFromServer(server)
+        try:
+            params.updateFromServer(server)
+        except Exception as e:
+            logger.fatal("Fetching command line: %s", e)
+            return 1
         cmdline = params.parseActions()
         if not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
@@ -579,7 +619,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]:
             universe = True
 
-        ret, error = server.runCommand(cmdline['action'])
+        try:
+            ret, error = server.runCommand(cmdline['action'])
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure
+            logger.fatal("Command '{}' failed: %s".format(cmdline), e)
+            return 1
         if error:
             logger.error("Command '%s' failed: %s" % (cmdline, error))
             return 1
@@ -597,26 +642,40 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     warnings = 0
     taskfailures = []
 
-    printinterval = 5000
-    lastprint = time.time()
+    printintervaldelta = 10 * 60 # 10 minutes
+    printinterval = printintervaldelta
+    pinginterval = 1 * 60 # 1 minute
+    lastevent = lastprint = time.time()
 
     termfilter = tf(main, helper, console_handlers, params.options.quiet)
     atexit.register(termfilter.finish)
 
-    while True:
+    # shutdown levels
+    # 0 - normal operation
+    # 1 - no new task execution, let current running tasks finish
+    # 2 - interrupting currently executing tasks
+    # 3 - we're done, exit
+    while main.shutdown < 3:
         try:
             if (lastprint + printinterval) <= time.time():
                 termfilter.keepAlive(printinterval)
-                printinterval += 5000
+                printinterval += printintervaldelta
             event = eventHandler.waitEvent(0)
             if event is None:
-                if main.shutdown > 1:
-                    break
+                if (lastevent + pinginterval) <= time.time():
+                    ret, error = server.runCommand(["ping"])
+                    if error or not ret:
+                        termfilter.clearFooter()
+                        print("No reply after pinging server (%s, %s), exiting." % (str(error), str(ret)))
+                        return_value = 3
+                        main.shutdown = 3
+                    lastevent = time.time()
                 if not parseprogress:
                     termfilter.updateFooter()
                 event = eventHandler.waitEvent(0.25)
                 if event is None:
                     continue
+            lastevent = time.time()
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
                 if not main.shutdown:
@@ -638,8 +697,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             if isinstance(event, logging.LogRecord):
                 lastprint = time.time()
-                printinterval = 5000
-                if event.levelno >= bb.msg.BBLogFormatter.ERROR:
+                printinterval = printintervaldelta
+                if event.levelno >= bb.msg.BBLogFormatter.ERRORONCE:
                     errors = errors + 1
                     return_value = 1
                 elif event.levelno == bb.msg.BBLogFormatter.WARNING:
@@ -653,10 +712,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN:
+                    if event.taskpid in helper.pidmap and event.levelno not in [bb.msg.BBLogFormatter.PLAIN, bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
-                if hasattr(event, 'fn'):
+                if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                     event.msg = event.fn + ': ' + event.msg
                 logging.getLogger(event.name).handle(event)
                 continue
@@ -721,15 +780,15 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 if event.error:
                     errors = errors + 1
                     logger.error(str(event))
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, bb.command.CommandExit):
                 if not return_value:
                     return_value = event.exitcode
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, bb.event.MultipleProviders):
                 logger.info(str(event))
@@ -745,7 +804,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
-                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
+                logger.info("Running setscene task %d of %d (%s)" % (event.stats.setscene_covered + event.stats.setscene_active + event.stats.setscene_notcovered + 1, event.stats.setscene_total, event.taskstring))
                 continue
 
             if isinstance(event, bb.runqueue.runQueueTaskStarted):
@@ -814,15 +873,26 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             logger.error("Unknown event: %s", event)
 
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure, don't attempt further comms and exit
+            logger.fatal("Executing event: %s", e)
+            return_value = 1
+            errors = errors + 1
+            main.shutdown = 3
         except EnvironmentError as ioerror:
             termfilter.clearFooter()
             # ignore interrupted io
             if ioerror.args[0] == 4:
                 continue
             sys.stderr.write(str(ioerror))
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shutdown = 3
         except KeyboardInterrupt:
             termfilter.clearFooter()
             if params.observe_only:
@@ -831,9 +901,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             def state_force_shutdown():
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                _, error = server.runCommand(["stateForceShutdown"])
-                if error:
-                    logger.error("Unable to cleanly stop: %s" % error)
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    if error:
+                        logger.error("Unable to cleanly stop: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly stop: %s", e)
 
             if not params.observe_only and main.shutdown == 1:
                 state_force_shutdown()
@@ -846,17 +920,24 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     _, error = server.runCommand(["stateShutdown"])
                     if error:
                         logger.error("Unable to cleanly shutdown: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly shutdown: %s", e)
                 except KeyboardInterrupt:
                     state_force_shutdown()
 
             main.shutdown = main.shutdown + 1
-            pass
         except Exception as e:
             import traceback
             sys.stderr.write(traceback.format_exc())
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shudown = 3
             return_value = 1
     try:
         termfilter.clearFooter()
@@ -867,11 +948,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             for failure in taskfailures:
                 summary += "\n  %s" % failure
         if warnings:
-            summary += pluralise("\nSummary: There was %s WARNING message shown.",
-                                 "\nSummary: There were %s WARNING messages shown.", warnings)
+            summary += pluralise("\nSummary: There was %s WARNING message.",
+                                 "\nSummary: There were %s WARNING messages.", warnings)
         if return_value and errors:
-            summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
-                                 "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
+            summary += pluralise("\nSummary: There was %s ERROR message, returning a non-zero exit code.",
+                                 "\nSummary: There were %s ERROR messages, returning a non-zero exit code.", errors)
         if summary and params.options.quiet == 0:
             print(summary)
 
diff --git a/bitbake/lib/bb/ui/ncurses.py b/bitbake/lib/bb/ui/ncurses.py
index cf1c876a51..18a706547a 100644
--- a/bitbake/lib/bb/ui/ncurses.py
+++ b/bitbake/lib/bb/ui/ncurses.py
@@ -227,6 +227,9 @@ class NCursesUI:
         shutdown = 0
 
         try:
+            if not params.observe_only:
+                params.updateToServer(server, os.environ.copy())
+
             params.updateFromServer(server)
             cmdline = params.parseActions()
             if not cmdline:
diff --git a/bitbake/lib/bb/ui/taskexp.py b/bitbake/lib/bb/ui/taskexp.py
index 2b246710ca..bedfd69b09 100644
--- a/bitbake/lib/bb/ui/taskexp.py
+++ b/bitbake/lib/bb/ui/taskexp.py
@@ -8,6 +8,7 @@
 #
 
 import sys
+import traceback
 
 try:
     import gi
@@ -176,7 +177,7 @@ class gtkthread(threading.Thread):
     quit = threading.Event()
     def __init__(self, shutdown):
         threading.Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.shutdown = shutdown
         if not Gtk.init_check()[0]:
             sys.stderr.write("Gtk+ init failed. Make sure DISPLAY variable is set.\n")
@@ -196,6 +197,7 @@ def main(server, eventHandler, params):
     gtkgui.start()
 
     try:
+        params.updateToServer(server, os.environ.copy())
         params.updateFromServer(server)
         cmdline = params.parseActions()
         if not cmdline:
@@ -218,6 +220,9 @@ def main(server, eventHandler, params):
     except client.Fault as x:
         print("XMLRPC Fault getting commandline:\n %s" % x)
         return
+    except Exception as e:
+        print("Exception in startup:\n %s" % traceback.format_exc())
+        return
 
     if gtkthread.quit.isSet():
         return
diff --git a/bitbake/lib/bb/ui/taskexp_ncurses.py b/bitbake/lib/bb/ui/taskexp_ncurses.py
new file mode 100755
index 0000000000..ea94a4987f
--- /dev/null
+++ b/bitbake/lib/bb/ui/taskexp_ncurses.py
@@ -0,0 +1,1511 @@
+#
+# BitBake Graphical ncurses-based Dependency Explorer
+#   * Based on the GTK implementation
+#   * Intended to run on any Linux host
+#
+# Copyright (C) 2007        Ross Burton
+# Copyright (C) 2007 - 2008 Richard Purdie
+# Copyright (C) 2022 - 2024 David Reyna
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+#
+# Execution example:
+#   $ bitbake -g -u taskexp_ncurses zlib acl
+#
+# Self-test example (executes a script of GUI actions):
+#   $ TASK_EXP_UNIT_TEST=1 bitbake -g -u taskexp_ncurses zlib acl
+#   ...
+#   $ echo $?
+#   0
+#   $ TASK_EXP_UNIT_TEST=1 bitbake -g -u taskexp_ncurses zlib acl foo
+#   ERROR: Nothing PROVIDES 'foo'. Close matches:
+#   ofono
+#   $ echo $?
+#   1
+#
+# Self-test with no terminal example (only tests dependency fetch from bitbake):
+#   $ TASK_EXP_UNIT_TEST_NOTERM=1 bitbake -g -u taskexp_ncurses quilt
+#   $ echo $?
+#   0
+#
+# Features:
+# * Ncurses is used for the presentation layer. Only the 'curses'
+#   library is used (none of the extension libraries), plus only
+#   one main screen is used (no sub-windows)
+# * Uses the 'generateDepTreeEvent' bitbake event to fetch the
+#   dynamic dependency data based on passed recipes
+# * Computes and provides reverse dependencies
+# * Supports task sorting on:
+#   (a) Task dependency order within each recipe
+#   (b) Pure alphabetical order
+#   (c) Provisions for third sort order (bitbake order?)
+# * The 'Filter' does a "*string*" wildcard filter on tasks in the
+#   main window, dynamically re-ordering and re-centering the content
+# * A 'Print' function exports the selected task or its whole recipe
+#   task set to the default file "taskdep.txt"
+# * Supports a progress bar for bitbake loads and file printing
+# * Line art for box drawing supported, ASCII art an alernative
+# * No horizontal scrolling support. Selected task's full name
+#   shown in bottom bar
+# * Dynamically catches terminals that are (or become) too small
+# * Exception to insure return to normal terminal on errors
+# * Debugging support, self test option
+#
+
+import sys
+import traceback
+import curses
+import re
+import time
+
+# Bitbake server support
+import threading
+from xmlrpc import client
+import bb
+import bb.event
+
+# Dependency indexes (depends_model)
+(TYPE_DEP, TYPE_RDEP) = (0, 1)
+DEPENDS_TYPE = 0
+DEPENDS_TASK = 1
+DEPENDS_DEPS = 2
+# Task indexes (task_list)
+TASK_NAME = 0
+TASK_PRIMARY = 1
+TASK_SORT_ALPHA = 2
+TASK_SORT_DEPS = 3
+TASK_SORT_BITBAKE = 4
+# Sort options (default is SORT_DEPS)
+SORT_ALPHA = 0
+SORT_DEPS = 1
+SORT_BITBAKE_ENABLE = False # NOTE: future sort
+SORT_BITBAKE = 2
+sort_model = SORT_DEPS
+# Print options
+PRINT_MODEL_1 = 0
+PRINT_MODEL_2 = 1
+print_model = PRINT_MODEL_2
+print_file_name = "taskdep_print.log"
+print_file_backup_name = "taskdep_print_backup.log"
+is_printed = False
+is_filter = False
+
+# Standard (and backup) key mappings
+CHAR_NUL = 0            # Used as self-test nop char
+CHAR_BS_H = 8           # Alternate backspace key
+CHAR_TAB = 9
+CHAR_RETURN = 10
+CHAR_ESCAPE = 27
+CHAR_UP = ord('{')      # Used as self-test ASCII char
+CHAR_DOWN = ord('}')    # Used as self-test ASCII char
+
+# Color_pair IDs
+CURSES_NORMAL = 0
+CURSES_HIGHLIGHT = 1
+CURSES_WARNING = 2
+
+
+#################################################
+### Debugging support
+###
+
+verbose = False
+
+# Debug: message display slow-step through display update issues
+def alert(msg,screen):
+    if msg:
+        screen.addstr(0, 10, '[%-4s]' % msg)
+        screen.refresh();
+        curses.napms(2000)
+    else:
+        if do_line_art:
+            for i in range(10, 24):
+                screen.addch(0, i, curses.ACS_HLINE)
+        else:
+            screen.addstr(0, 10, '-' * 14)
+        screen.refresh();
+
+# Debug: display edge conditions on frame movements
+def debug_frame(nbox_ojb):
+    if verbose:
+        nbox_ojb.screen.addstr(0, 50, '[I=%2d,O=%2d,S=%3s,H=%2d,M=%4d]' % (
+            nbox_ojb.cursor_index,
+            nbox_ojb.cursor_offset,
+            nbox_ojb.scroll_offset,
+            nbox_ojb.inside_height,
+            len(nbox_ojb.task_list),
+        ))
+        nbox_ojb.screen.refresh();
+
+#
+# Unit test (assumes that 'quilt-native' is always present)
+#
+
+unit_test = os.environ.get('TASK_EXP_UNIT_TEST')
+unit_test_cmnds=[
+    '# Default selected task in primary box',
+    'tst_selected=<TASK>.do_recipe_qa',
+    '# Default selected task in deps',
+    'tst_entry=<TAB>',
+    'tst_selected=',
+    '# Default selected task in rdeps',
+    'tst_entry=<TAB>',
+    'tst_selected=<TASK>.do_fetch',
+    "# Test 'select' back to primary box",
+    'tst_entry=<CR>',
+    '#tst_entry=<DOWN>',  # optional injected error
+    'tst_selected=<TASK>.do_fetch',
+    '# Check filter',
+    'tst_entry=/uilt-nativ/',
+    'tst_selected=quilt-native.do_recipe_qa',
+    '# Check print',
+    'tst_entry=p',
+    'tst_printed=quilt-native.do_fetch',
+    '#tst_printed=quilt-foo.do_nothing',  # optional injected error
+    '# Done!',
+    'tst_entry=q',
+]
+unit_test_idx=0
+unit_test_command_chars=''
+unit_test_results=[]
+def unit_test_action(active_package):
+    global unit_test_idx
+    global unit_test_command_chars
+    global unit_test_results
+    ret = CHAR_NUL
+    if unit_test_command_chars:
+        ch = unit_test_command_chars[0]
+        unit_test_command_chars = unit_test_command_chars[1:]
+        time.sleep(0.5)
+        ret = ord(ch)
+    else:
+        line = unit_test_cmnds[unit_test_idx]
+        unit_test_idx += 1
+        line = re.sub('#.*', '', line).strip()
+        line = line.replace('<TASK>',active_package.primary[0])
+        line = line.replace('<TAB>','\t').replace('<CR>','\n')
+        line = line.replace('<UP>','{').replace('<DOWN>','}')
+        if not line: line = 'nop=nop'
+        cmnd,value = line.split('=')
+        if cmnd == 'tst_entry':
+            unit_test_command_chars = value
+        elif cmnd == 'tst_selected':
+            active_selected = active_package.get_selected()
+            if active_selected != value:
+                unit_test_results.append("ERROR:SELFTEST:expected '%s' but got '%s' (NOTE:bitbake may have changed)" % (value,active_selected))
+                ret = ord('Q')
+            else:
+                unit_test_results.append("Pass:SELFTEST:found '%s'" % (value))
+        elif cmnd == 'tst_printed':
+            result = os.system('grep %s %s' % (value,print_file_name))
+            if result:
+                unit_test_results.append("ERROR:PRINTTEST:expected '%s' in '%s'" % (value,print_file_name))
+                ret = ord('Q')
+            else:
+                unit_test_results.append("Pass:PRINTTEST:found '%s'" % (value))
+    # Return the action (CHAR_NUL for no action til next round)
+    return(ret)
+
+# Unit test without an interative terminal (e.g. ptest)
+unit_test_noterm = os.environ.get('TASK_EXP_UNIT_TEST_NOTERM')
+
+
+#################################################
+### Window frame rendering
+###
+### By default, use the normal line art. Since
+###  these extended characters are not ASCII, one
+###  must use the ncursus API to render them
+### The alternate ASCII line art set is optionally
+###  available via the 'do_line_art' flag
+
+# By default, render frames using line art
+do_line_art = True
+
+# ASCII render set option
+CHAR_HBAR = '-'
+CHAR_VBAR = '|'
+CHAR_UL_CORNER = '/'
+CHAR_UR_CORNER = '\\'
+CHAR_LL_CORNER = '\\'
+CHAR_LR_CORNER = '/'
+
+# Box frame drawing with line-art
+def line_art_frame(box):
+    x = box.base_x
+    y = box.base_y
+    w = box.width
+    h = box.height + 1
+
+    if do_line_art:
+        for i in range(1, w - 1):
+            box.screen.addch(y, x + i, curses.ACS_HLINE, box.color)
+            box.screen.addch(y + h - 1, x + i, curses.ACS_HLINE, box.color)
+        body_line = "%s" % (' ' * (w - 2))
+        for i in range(1, h - 1):
+            box.screen.addch(y + i, x, curses.ACS_VLINE, box.color)
+            box.screen.addstr(y + i, x + 1, body_line, box.color)
+            box.screen.addch(y + i, x + w - 1, curses.ACS_VLINE, box.color)
+        box.screen.addch(y, x, curses.ACS_ULCORNER, box.color)
+        box.screen.addch(y, x + w - 1, curses.ACS_URCORNER, box.color)
+        box.screen.addch(y + h - 1, x, curses.ACS_LLCORNER, box.color)
+        box.screen.addch(y + h - 1, x + w - 1, curses.ACS_LRCORNER, box.color)
+    else:
+        top_line  = "%s%s%s" % (CHAR_UL_CORNER,CHAR_HBAR * (w - 2),CHAR_UR_CORNER)
+        body_line = "%s%s%s" % (CHAR_VBAR,' ' * (w - 2),CHAR_VBAR)
+        bot_line  = "%s%s%s" % (CHAR_UR_CORNER,CHAR_HBAR * (w - 2),CHAR_UL_CORNER)
+        tag_line  = "%s%s%s" % ('[',CHAR_HBAR * (w - 2),']')
+        # Top bar
+        box.screen.addstr(y, x, top_line)
+        # Middle frame
+        for i in range(1, (h - 1)):
+            box.screen.addstr(y+i, x, body_line)
+        # Bottom bar
+        box.screen.addstr(y + (h - 1), x, bot_line)
+
+# Connect the separate boxes
+def line_art_fixup(box):
+    if do_line_art:
+        box.screen.addch(box.base_y+2, box.base_x, curses.ACS_LTEE, box.color)
+        box.screen.addch(box.base_y+2, box.base_x+box.width-1, curses.ACS_RTEE, box.color)
+
+
+#################################################
+### Ncurses box object : box frame object to display
+### and manage a sub-window's display elements
+### using basic ncurses
+###
+### Supports:
+###   * Frame drawing, content (re)drawing
+###   * Content scrolling via ArrowUp, ArrowDn, PgUp, PgDN,
+###   * Highlighting for active selected item
+###   * Content sorting based on selected sort model
+###
+
+class NBox():
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        # Box description
+        self.screen = screen
+        self.label = label
+        self.primary = primary
+        self.color = curses.color_pair(CURSES_NORMAL) if screen else None
+        # Box boundaries
+        self.base_x = base_x
+        self.base_y = base_y
+        self.width = width
+        self.height = height
+        # Cursor/scroll management
+        self.cursor_enable = False
+        self.cursor_index = 0   # Absolute offset
+        self.cursor_offset = 0  # Frame centric offset
+        self.scroll_offset = 0  # Frame centric offset
+        # Box specific content
+        # Format of each entry is [package_name,is_primary_recipe,alpha_sort_key,deps_sort_key]
+        self.task_list = []
+
+    @property
+    def inside_width(self):
+        return(self.width-2)
+
+    @property
+    def inside_height(self):
+        return(self.height-2)
+
+    # Populate the box's content, include the sort mappings and is_primary flag
+    def task_list_append(self,task_name,dep):
+        task_sort_alpha = task_name
+        task_sort_deps = dep.get_dep_sort(task_name)
+        is_primary = False
+        for primary in self.primary:
+            if task_name.startswith(primary+'.'):
+                is_primary = True
+        if SORT_BITBAKE_ENABLE:
+            task_sort_bitbake = dep.get_bb_sort(task_name)
+            self.task_list.append([task_name,is_primary,task_sort_alpha,task_sort_deps,task_sort_bitbake])
+        else:
+            self.task_list.append([task_name,is_primary,task_sort_alpha,task_sort_deps])
+
+    def reset(self):
+        self.task_list = []
+        self.cursor_index = 0   # Absolute offset
+        self.cursor_offset = 0  # Frame centric offset
+        self.scroll_offset = 0  # Frame centric offset
+
+    # Sort the box's content based on the current sort model
+    def sort(self):
+        if SORT_ALPHA == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_ALPHA])
+        elif SORT_DEPS == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_DEPS])
+        elif SORT_BITBAKE == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_BITBAKE])
+
+    # The target package list (to hightlight), from the command line
+    def set_primary(self,primary):
+        self.primary = primary
+
+    # Draw the box's outside frame
+    def draw_frame(self):
+        line_art_frame(self)
+        # Title
+        self.screen.addstr(self.base_y,
+            (self.base_x + (self.width//2))-((len(self.label)+2)//2),
+            '['+self.label+']')
+        self.screen.refresh()
+
+    # Draw the box's inside text content
+    def redraw(self):
+        task_list_len = len(self.task_list)
+        # Middle frame
+        body_line = "%s" % (' ' * (self.inside_width-1) )
+        for i in range(0,self.inside_height+1):
+            if i < (task_list_len + self.scroll_offset):
+                str_ctl = "%%-%ss" % (self.width-3)
+                # Safety assert
+                if (i + self.scroll_offset) >= task_list_len:
+                    alert("REDRAW:%2d,%4d,%4d" % (i,self.scroll_offset,task_list_len),self.screen)
+                    break
+
+                task_obj = self.task_list[i + self.scroll_offset]
+                task = task_obj[TASK_NAME][:self.inside_width-1]
+                task_primary = task_obj[TASK_PRIMARY]
+
+                if task_primary:
+                    line = str_ctl % task[:self.inside_width-1]
+                    self.screen.addstr(self.base_y+1+i, self.base_x+2, line, curses.A_BOLD)
+                else:
+                    line = str_ctl % task[:self.inside_width-1]
+                    self.screen.addstr(self.base_y+1+i, self.base_x+2, line)
+            else:
+                line = "%s" % (' ' * (self.inside_width-1) )
+                self.screen.addstr(self.base_y+1+i, self.base_x+2, line)
+        self.screen.refresh()
+
+    # Show the current selected task over the bottom of the frame
+    def show_selected(self,selected_task):
+        if not selected_task:
+            selected_task = self.get_selected()
+        tag_line   = "%s%s%s" % ('[',CHAR_HBAR * (self.width-2),']')
+        self.screen.addstr(self.base_y + self.height, self.base_x, tag_line)
+        self.screen.addstr(self.base_y + self.height,
+            (self.base_x + (self.width//2))-((len(selected_task)+2)//2),
+            '['+selected_task+']')
+        self.screen.refresh()
+
+    # Load box with new table of content
+    def update_content(self,task_list):
+        self.task_list = task_list
+        if self.cursor_enable:
+            cursor_update(turn_on=False)
+        self.cursor_index = 0
+        self.cursor_offset = 0
+        self.scroll_offset = 0
+        self.redraw()
+        if self.cursor_enable:
+            cursor_update(turn_on=True)
+
+    # Manage the box's highlighted task and blinking cursor character
+    def cursor_on(self,is_on):
+        self.cursor_enable = is_on
+        self.cursor_update(is_on)
+
+    # High-light the current pointed package, normal for released packages
+    def cursor_update(self,turn_on=True):
+        str_ctl = "%%-%ss" % (self.inside_width-1)
+        try:
+            if len(self.task_list):
+                task_obj = self.task_list[self.cursor_index]
+                task = task_obj[TASK_NAME][:self.inside_width-1]
+                task_primary = task_obj[TASK_PRIMARY]
+                task_font = curses.A_BOLD if task_primary else 0
+            else:
+                task = ''
+                task_font = 0
+        except Exception as e:
+            alert("CURSOR_UPDATE:%s" % (e),self.screen)
+            return
+        if turn_on:
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+1,">", curses.color_pair(CURSES_HIGHLIGHT) | curses.A_BLINK)
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+2,str_ctl % task, curses.color_pair(CURSES_HIGHLIGHT) | task_font)
+        else:
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+1," ")
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+2,str_ctl % task, task_font)
+
+    # Down arrow
+    def line_down(self):
+        if len(self.task_list) <= (self.cursor_index+1):
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index += 1
+        self.cursor_offset += 1
+        if self.cursor_offset > (self.inside_height):
+            self.cursor_offset -= 1
+            self.scroll_offset += 1
+            self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Up arrow
+    def line_up(self):
+        if 0 > (self.cursor_index-1):
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index -= 1
+        self.cursor_offset -= 1
+        if self.cursor_offset < 0:
+            self.cursor_offset += 1
+            self.scroll_offset -= 1
+            self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Page down
+    def page_down(self):
+        max_task = len(self.task_list)-1
+        if max_task < self.inside_height:
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index += 10
+        self.cursor_index = min(self.cursor_index,max_task)
+        self.cursor_offset = min(self.inside_height,self.cursor_index)
+        self.scroll_offset = self.cursor_index - self.cursor_offset
+        self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Page up
+    def page_up(self):
+        max_task = len(self.task_list)-1
+        if max_task < self.inside_height:
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index -= 10
+        self.cursor_index = max(self.cursor_index,0)
+        self.cursor_offset = max(0, self.inside_height - (max_task - self.cursor_index))
+        self.scroll_offset = self.cursor_index - self.cursor_offset
+        self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Return the currently selected task name for this box
+    def get_selected(self):
+        if self.task_list:
+            return(self.task_list[self.cursor_index][TASK_NAME])
+        else:
+            return('')
+
+#################################################
+### The helper sub-windows
+###
+
+# Show persistent help at the top of the screen
+class HelpBarView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(HelpBarView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    def show_help(self,show):
+        self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.inside_width))
+        if show:
+            help = "Help='?' Filter='/' NextBox=<Tab> Select=<Enter> Print='p','P' Quit='q'"
+            bar_size = self.inside_width - 5 - len(help)
+            self.screen.addstr(self.base_y,self.base_x+((self.inside_width-len(help))//2), help)
+        self.screen.refresh()
+
+# Pop up a detailed Help box
+class HelpBoxView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height, dep):
+        super(HelpBoxView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+        self.x_pos = 0
+        self.y_pos = 0
+        self.dep = dep
+
+    # Instantial the pop-up help box
+    def show_help(self,show):
+        self.x_pos = self.base_x + 4
+        self.y_pos = self.base_y + 2
+
+        def add_line(line):
+            if line:
+                self.screen.addstr(self.y_pos,self.x_pos,line)
+            self.y_pos += 1
+
+        # Gather some statisics
+        dep_count = 0
+        rdep_count = 0
+        for task_obj in self.dep.depends_model:
+            if TYPE_DEP == task_obj[DEPENDS_TYPE]:
+                dep_count += 1
+            elif TYPE_RDEP == task_obj[DEPENDS_TYPE]:
+                rdep_count += 1
+
+        self.draw_frame()
+        line_art_fixup(self.dep)
+        add_line("Quit                : 'q' ")
+        add_line("Filter task names   : '/'")
+        add_line("Tab to next box     : <Tab>")
+        add_line("Select a task       : <Enter>")
+        add_line("Print task's deps   : 'p'")
+        add_line("Print recipe's deps : 'P'")
+        add_line(" -> '%s'" % print_file_name)
+        add_line("Sort toggle         : 's'")
+        add_line(" %s Recipe inner-depends order" % ('->' if (SORT_DEPS == sort_model) else '- '))
+        add_line(" %s Alpha-numeric order" % ('->' if (SORT_ALPHA == sort_model) else '- '))
+        if SORT_BITBAKE_ENABLE:
+            add_line(" %s Bitbake order" % ('->' if (TASK_SORT_BITBAKE == sort_model) else '- '))
+        add_line("Alternate backspace : <CTRL-H>")
+        add_line("")
+        add_line("Primary recipes = %s"  % ','.join(self.primary))
+        add_line("Task  count     = %4d" % len(self.dep.pkg_model))
+        add_line("Deps  count     = %4d" % dep_count)
+        add_line("RDeps count     = %4d" % rdep_count)
+        add_line("")
+        self.screen.addstr(self.y_pos,self.x_pos+7,"<Press any key>", curses.color_pair(CURSES_HIGHLIGHT))
+        self.screen.refresh()
+        c = self.screen.getch()
+
+# Show a progress bar
+class ProgressView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(ProgressView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    def progress(self,title,current,max):
+        if title:
+            self.label = title
+        else:
+            title = self.label
+        if max <=0: max = 10
+        bar_size = self.width - 7 - len(title)
+        bar_done = int( (float(current)/float(max)) * float(bar_size) )
+        self.screen.addstr(self.base_y,self.base_x, " %s:[%s%s]" % (title,'*' * bar_done,' ' * (bar_size-bar_done)))
+        self.screen.refresh()
+        return(current+1)
+
+    def clear(self):
+        self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.width))
+        self.screen.refresh()
+
+# Implement a task filter bar
+class FilterView(NBox):
+    SEARCH_NOP = 0
+    SEARCH_GO = 1
+    SEARCH_CANCEL = 2
+
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(FilterView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+        self.do_show = False
+        self.filter_str = ""
+
+    def clear(self,enable_show=True):
+        self.filter_str = ""
+
+    def show(self,enable_show=True):
+        self.do_show = enable_show
+        if self.do_show:
+            self.screen.addstr(self.base_y,self.base_x, "[ Filter: %-25s ]      '/'=cancel, format='abc'       " % self.filter_str[0:25])
+        else:
+            self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.width))
+        self.screen.refresh()
+
+    def show_prompt(self):
+        self.screen.addstr(self.base_y,self.base_x + 10 + len(self.filter_str), " ")
+        self.screen.addstr(self.base_y,self.base_x + 10 + len(self.filter_str), "")
+
+    # Keys specific to the filter box (start/stop filter keys are in the main loop)
+    def input(self,c,ch):
+        ret = self.SEARCH_GO
+        if c in (curses.KEY_BACKSPACE,CHAR_BS_H):
+            #  Backspace
+            if self.filter_str:
+                self.filter_str = self.filter_str[0:-1]
+                self.show()
+        elif ((ch >= 'a') and (ch <= 'z')) or ((ch >= 'A') and (ch <= 'Z')) or ((ch >= '0') and (ch <= '9')) or (ch in (' ','_','.','-')):
+            # The isalnum() acts strangly with keypad(True), so explicit bounds
+            self.filter_str += ch
+            self.show()
+        else:
+            ret = self.SEARCH_NOP
+        return(ret)
+
+
+#################################################
+### The primary dependency windows
+###
+
+# The main list of package tasks
+class PackageView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    # Find and verticaly center a selected task (from filter or from dependent box)
+    # The 'task_filter_str' can be a full or a partial (filter) task name
+    def find(self,task_filter_str):
+        found = False
+        max = self.height-2
+        if not task_filter_str:
+            return(found)
+        for i,task_obj in enumerate(self.task_list):
+            task = task_obj[TASK_NAME]
+            if task.startswith(task_filter_str):
+                self.cursor_on(False)
+                self.cursor_index = i
+
+                # Position selected at vertical center
+                vcenter = self.inside_height // 2
+                if self.cursor_index <= vcenter:
+                    self.scroll_offset = 0
+                    self.cursor_offset = self.cursor_index
+                elif self.cursor_index >= (len(self.task_list) - vcenter - 1):
+                    self.cursor_offset = self.inside_height-1
+                    self.scroll_offset = self.cursor_index - self.cursor_offset
+                else:
+                    self.cursor_offset = vcenter
+                    self.scroll_offset = self.cursor_index - self.cursor_offset
+
+                self.redraw()
+                self.cursor_on(True)
+                found = True
+                break
+        return(found)
+
+# The view of dependent packages
+class PackageDepView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageDepView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+# The view of reverse-dependent packages
+class PackageReverseDepView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageReverseDepView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+
+#################################################
+### DepExplorer : The parent frame and object
+###
+
+class DepExplorer(NBox):
+    def __init__(self,screen):
+        title = "Task Dependency Explorer"
+        super(DepExplorer, self).__init__(screen, 'Task Dependency Explorer','',0,0,80,23)
+
+        self.screen = screen
+        self.pkg_model = []
+        self.depends_model = []
+        self.dep_sort_map = {}
+        self.bb_sort_map = {}
+        self.filter_str = ''
+        self.filter_prev = 'deadbeef'
+
+        if self.screen:
+            self.help_bar_view = HelpBarView(screen, "Help",'',1,1,79,1)
+            self.help_box_view = HelpBoxView(screen, "Help",'',0,2,40,20,self)
+            self.progress_view = ProgressView(screen, "Progress",'',2,1,76,1)
+            self.filter_view = FilterView(screen, "Filter",'',2,1,76,1)
+            self.package_view = PackageView(screen, "Package",'alpha', 0,2,40,20)
+            self.dep_view = PackageDepView(screen, "Dependencies",'beta',40,2,40,10)
+            self.reverse_view = PackageReverseDepView(screen, "Dependent Tasks",'gamma',40,13,40,9)
+            self.draw_frames()
+
+    # Draw this main window's frame and all sub-windows
+    def draw_frames(self):
+        self.draw_frame()
+        self.package_view.draw_frame()
+        self.dep_view.draw_frame()
+        self.reverse_view.draw_frame()
+        if is_filter:
+            self.filter_view.show(True)
+            self.filter_view.show_prompt()
+        else:
+            self.help_bar_view.show_help(True)
+        self.package_view.redraw()
+        self.dep_view.redraw()
+        self.reverse_view.redraw()
+        self.show_selected(self.package_view.get_selected())
+        line_art_fixup(self)
+
+    # Parse the bitbake dependency event object
+    def parse(self, depgraph):
+        for task in depgraph["tdepends"]:
+            self.pkg_model.insert(0, task)
+            for depend in depgraph["tdepends"][task]:
+                self.depends_model.insert (0, (TYPE_DEP, task, depend))
+                self.depends_model.insert (0, (TYPE_RDEP, depend, task))
+        if self.screen:
+            self.dep_sort_prep()
+
+    # Prepare the dependency sort order keys
+    # This method creates sort keys per recipe tasks in
+    # the order of each recipe's internal dependecies
+    # Method:
+    #   Filter the tasks in dep order in dep_sort_map = {}
+    #   (a) Find a task that has no dependecies
+    #       Ignore non-recipe specific tasks
+    #   (b) Add it to the sort mapping dict with
+    #       key of "<task_group>_<order>"
+    #   (c) Remove it as a dependency from the other tasks
+    #   (d) Repeat till all tasks are mapped
+    # Use placeholders to insure each sub-dict is instantiated
+    def dep_sort_prep(self):
+        self.progress_view.progress('DepSort',0,4)
+        # Init the task base entries
+        self.progress_view.progress('DepSort',1,4)
+        dep_table = {}
+        bb_index = 0
+        for task in self.pkg_model:
+            # First define the incoming bitbake sort order
+            self.bb_sort_map[task] = "%04d" % (bb_index)
+            bb_index += 1
+            task_group = task[0:task.find('.')]
+            if task_group not in dep_table:
+                dep_table[task_group] = {}
+                dep_table[task_group]['-'] = {}         # Placeholder
+            if task not in dep_table[task_group]:
+                dep_table[task_group][task] = {}
+                dep_table[task_group][task]['-'] = {}   # Placeholder
+        # Add the task dependecy entries
+        self.progress_view.progress('DepSort',2,4)
+        for task_obj in self.depends_model:
+            if task_obj[DEPENDS_TYPE] != TYPE_DEP:
+                continue
+            task = task_obj[DEPENDS_TASK]
+            task_dep = task_obj[DEPENDS_DEPS]
+            task_group = task[0:task.find('.')]
+            # Only track depends within same group
+            if task_dep.startswith(task_group+'.'):
+                dep_table[task_group][task][task_dep] = 1
+        self.progress_view.progress('DepSort',3,4)
+        for task_group in dep_table:
+            dep_index = 0
+            # Whittle down the tasks of each group
+            this_pass = 1
+            do_loop = True
+            while (len(dep_table[task_group]) > 1) and do_loop:
+                this_pass += 1
+                is_change = False
+                delete_list = []
+                for task in dep_table[task_group]:
+                    if '-' == task:
+                        continue
+                    if 1 == len(dep_table[task_group][task]):
+                        is_change = True
+                        # No more deps, so collect this task...
+                        self.dep_sort_map[task] = "%s_%04d" % (task_group,dep_index)
+                        dep_index += 1
+                        # ... remove it from other lists as resolved ...
+                        for dep_task in dep_table[task_group]:
+                            if task in dep_table[task_group][dep_task]:
+                                del dep_table[task_group][dep_task][task]
+                        # ... and remove it from from the task group
+                        delete_list.append(task)
+                for task in delete_list:
+                    del dep_table[task_group][task]
+                if not is_change:
+                    alert("ERROR:DEP_SIEVE_NO_CHANGE:%s" % task_group,self.screen)
+                    do_loop = False
+                    continue
+        self.progress_view.progress('',4,4)
+        self.progress_view.clear()
+        self.help_bar_view.show_help(True)
+        if len(self.dep_sort_map) != len(self.pkg_model):
+            alert("ErrorDepSort:%d/%d" % (len(self.dep_sort_map),len(self.pkg_model)),self.screen)
+
+    # Look up a dep sort order key
+    def get_dep_sort(self,key):
+        if key in self.dep_sort_map:
+            return(self.dep_sort_map[key])
+        else:
+            return(key)
+
+    # Look up a bitbake sort order key
+    def get_bb_sort(self,key):
+        if key in self.bb_sort_map:
+            return(self.bb_sort_map[key])
+        else:
+            return(key)
+
+    # Find the selected package in the main frame, update the dependency frames content accordingly
+    def select(self, package_name, only_update_dependents=False):
+        if not package_name:
+            package_name = self.package_view.get_selected()
+        # alert("SELECT:%s:" % package_name,self.screen)
+
+        if self.filter_str != self.filter_prev:
+            self.package_view.cursor_on(False)
+            # Fill of the main package task list using new filter
+            self.package_view.task_list = []
+            for package in self.pkg_model:
+                if self.filter_str:
+                    if self.filter_str in package:
+                        self.package_view.task_list_append(package,self)
+                else:
+                    self.package_view.task_list_append(package,self)
+            self.package_view.sort()
+            self.filter_prev = self.filter_str
+
+            # Old position is lost, assert new position of previous task (if still filtered in)
+            self.package_view.cursor_index = 0
+            self.package_view.cursor_offset = 0
+            self.package_view.scroll_offset = 0
+            self.package_view.redraw()
+            self.package_view.cursor_on(True)
+
+        # Make sure the selected package is in view, with implicit redraw()
+        if (not only_update_dependents):
+            self.package_view.find(package_name)
+        # In case selected name change (i.e. filter removed previous)
+        package_name = self.package_view.get_selected()
+
+        # Filter the package's dependent list to the dependent view
+        self.dep_view.reset()
+        for package_def in self.depends_model:
+            if (package_def[DEPENDS_TYPE] == TYPE_DEP) and (package_def[DEPENDS_TASK] == package_name):
+                self.dep_view.task_list_append(package_def[DEPENDS_DEPS],self)
+        self.dep_view.sort()
+        self.dep_view.redraw()
+        # Filter the package's dependent list to the reverse dependent view
+        self.reverse_view.reset()
+        for package_def in self.depends_model:
+            if (package_def[DEPENDS_TYPE] == TYPE_RDEP) and (package_def[DEPENDS_TASK] == package_name):
+                self.reverse_view.task_list_append(package_def[DEPENDS_DEPS],self)
+        self.reverse_view.sort()
+        self.reverse_view.redraw()
+        self.show_selected(package_name)
+        self.screen.refresh()
+
+    # The print-to-file method
+    def print_deps(self,whole_group=False):
+        global is_printed
+        # Print the selected deptree(s) to a file
+        if not is_printed:
+            try:
+                # Move to backup any exiting file before first write
+                if os.path.isfile(print_file_name):
+                    os.system('mv -f %s %s' % (print_file_name,print_file_backup_name))
+            except Exception as e:
+                alert(e,self.screen)
+                alert('',self.screen)
+        print_list = []
+        selected_task = self.package_view.get_selected()
+        if not selected_task:
+            return
+        if not whole_group:
+            print_list.append(selected_task)
+        else:
+            # Use the presorted task_group order from 'package_view'
+            task_group = selected_task[0:selected_task.find('.')+1]
+            for task_obj in self.package_view.task_list:
+                task = task_obj[TASK_NAME]
+                if task.startswith(task_group):
+                    print_list.append(task)
+        with open(print_file_name, "a") as fd:
+            print_max = len(print_list)
+            print_count = 1
+            self.progress_view.progress('Write "%s"' % print_file_name,0,print_max)
+            for task in print_list:
+                print_count = self.progress_view.progress('',print_count,print_max)
+                self.select(task)
+                self.screen.refresh();
+                # Utilize the current print output model
+                if print_model == PRINT_MODEL_1:
+                    print("=== Dependendency Snapshot ===",file=fd)
+                    print(" = Package =",file=fd)
+                    print('   '+task,file=fd)
+                    # Fill in the matching dependencies
+                    print(" = Dependencies =",file=fd)
+                    for task_obj in self.dep_view.task_list:
+                        print('   '+ task_obj[TASK_NAME],file=fd)
+                    print(" = Dependent Tasks =",file=fd)
+                    for task_obj in self.reverse_view.task_list:
+                        print('   '+ task_obj[TASK_NAME],file=fd)
+                if print_model == PRINT_MODEL_2:
+                    print("=== Dependendency Snapshot ===",file=fd)
+                    dep_count = len(self.dep_view.task_list) - 1
+                    for i,task_obj in enumerate(self.dep_view.task_list):
+                        print('%s%s' % ("Dep    =" if (i==dep_count) else "        ",task_obj[TASK_NAME]),file=fd)
+                    if not self.dep_view.task_list:
+                        print('Dep    =',file=fd)
+                    print("Package=%s" % task,file=fd)
+                    for i,task_obj in enumerate(self.reverse_view.task_list):
+                        print('%s%s' % ("RDep   =" if (i==0) else "        ",task_obj[TASK_NAME]),file=fd)
+                    if not self.reverse_view.task_list:
+                        print('RDep   =',file=fd)
+                curses.napms(2000)
+                self.progress_view.clear()
+                self.help_bar_view.show_help(True)
+            print('',file=fd)
+        # Restore display to original selected task
+        self.select(selected_task)
+        is_printed = True
+
+#################################################
+### Load bitbake data
+###
+
+def bitbake_load(server, eventHandler, params, dep, curses_off, screen):
+    global bar_len_old
+    bar_len_old = 0
+
+    # Support no screen
+    def progress(msg,count,max):
+        global bar_len_old
+        if screen:
+            dep.progress_view.progress(msg,count,max)
+        else:
+            if msg:
+                if bar_len_old:
+                    bar_len_old = 0
+                    print("\n")
+                print(f"{msg}: ({count} of {max})")
+            else:
+                bar_len = int((count*40)/max)
+                if bar_len_old != bar_len:
+                    print(f"{'*' * (bar_len-bar_len_old)}",end='',flush=True)
+                bar_len_old = bar_len
+    def clear():
+        if screen:
+            dep.progress_view.clear()
+    def clear_curses(screen):
+        if screen:
+            curses_off(screen)
+
+    #
+    # Trigger bitbake "generateDepTreeEvent"
+    #
+
+    cmdline = ''
+    try:
+        params.updateToServer(server, os.environ.copy())
+        params.updateFromServer(server)
+        cmdline = params.parseActions()
+        if not cmdline:
+            clear_curses(screen)
+            print("ERROR: nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+            return 1,cmdline
+        if 'msg' in cmdline and cmdline['msg']:
+            clear_curses(screen)
+            print('ERROR: ' + cmdline['msg'])
+            return 1,cmdline
+        cmdline = cmdline['action']
+        if not cmdline or cmdline[0] != "generateDotGraph":
+            clear_curses(screen)
+            print("ERROR: This UI requires the -g option")
+            return 1,cmdline
+        ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]])
+        if error:
+            clear_curses(screen)
+            print("ERROR: running command '%s': %s" % (cmdline, error))
+            return 1,cmdline
+        elif not ret:
+            clear_curses(screen)
+            print("ERROR: running command '%s': returned %s" % (cmdline, ret))
+            return 1,cmdline
+    except client.Fault as x:
+        clear_curses(screen)
+        print("ERROR: XMLRPC Fault getting commandline:\n %s" % x)
+        return 1,cmdline
+    except Exception as e:
+        clear_curses(screen)
+        print("ERROR: in startup:\n %s" % traceback.format_exc())
+        return 1,cmdline
+
+    #
+    # Receive data from bitbake
+    #
+
+    progress_total = 0
+    load_bitbake = True
+    quit = False
+    try:
+        while load_bitbake:
+            try:
+                event = eventHandler.waitEvent(0.25)
+                if quit:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    clear_curses(screen)
+                    if error:
+                        print('Unable to cleanly stop: %s' % error)
+                    break
+
+                if event is None:
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadStarted):
+                    progress_total = event.total
+                    progress('Loading Cache',0,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadProgress):
+                    x = event.current
+                    progress('',x,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadCompleted):
+                    clear()
+                    progress('Bitbake... ',1,2)
+                    continue
+
+                if isinstance(event, bb.event.ParseStarted):
+                    progress_total = event.total
+                    progress('Processing recipes',0,progress_total)
+                    if progress_total == 0:
+                        continue
+
+                if isinstance(event, bb.event.ParseProgress):
+                    x = event.current
+                    progress('',x,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.ParseCompleted):
+                    progress('Generating dependency tree',0,3)
+                    continue
+
+                if isinstance(event, bb.event.DepTreeGenerated):
+                    progress('Generating dependency tree',1,3)
+                    dep.parse(event._depgraph)
+                    progress('Generating dependency tree',2,3)
+
+                if isinstance(event, bb.command.CommandCompleted):
+                    load_bitbake = False
+                    progress('Generating dependency tree',3,3)
+                    clear()
+                    if screen:
+                        dep.help_bar_view.show_help(True)
+                    continue
+
+                if isinstance(event, bb.event.NoProvider):
+                    clear_curses(screen)
+                    print('ERROR: %s' % event)
+
+                    _, error = server.runCommand(["stateShutdown"])
+                    if error:
+                        print('ERROR: Unable to cleanly shutdown: %s' % error)
+                    return 1,cmdline
+
+                if isinstance(event, bb.command.CommandFailed):
+                    clear_curses(screen)
+                    print('ERROR: ' + str(event))
+                    return event.exitcode,cmdline
+
+                if isinstance(event, bb.command.CommandExit):
+                    clear_curses(screen)
+                    return event.exitcode,cmdline
+
+                if isinstance(event, bb.cooker.CookerExit):
+                    break
+
+                continue
+            except EnvironmentError as ioerror:
+                # ignore interrupted io
+                if ioerror.args[0] == 4:
+                    pass
+            except KeyboardInterrupt:
+                if shutdown == 2:
+                    clear_curses(screen)
+                    print("\nThird Keyboard Interrupt, exit.\n")
+                    break
+                if shutdown == 1:
+                    clear_curses(screen)
+                    print("\nSecond Keyboard Interrupt, stopping...\n")
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    if error:
+                        print('Unable to cleanly stop: %s' % error)
+                if shutdown == 0:
+                    clear_curses(screen)
+                    print("\nKeyboard Interrupt, closing down...\n")
+                    _, error = server.runCommand(["stateShutdown"])
+                    if error:
+                        print('Unable to cleanly shutdown: %s' % error)
+                shutdown = shutdown + 1
+                pass
+    except Exception as e:
+        # Safe exit on error
+        clear_curses(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+
+    return 0,cmdline
+
+#################################################
+### main
+###
+
+SCREEN_COL_MIN = 83
+SCREEN_ROW_MIN = 26
+
+def main(server, eventHandler, params):
+    global verbose
+    global sort_model
+    global print_model
+    global is_printed
+    global is_filter
+    global screen_too_small
+
+    shutdown = 0
+    screen_too_small = False
+    quit = False
+
+    # Unit test with no terminal?
+    if unit_test_noterm:
+        # Load bitbake, test that there is valid dependency data, then exit
+        screen = None
+        print("* UNIT TEST:START")
+        dep = DepExplorer(screen)
+        print("* UNIT TEST:BITBAKE FETCH")
+        ret,cmdline = bitbake_load(server, eventHandler, params, dep, None, screen)
+        if ret:
+            print("* UNIT TEST: BITBAKE FAILED")
+            return ret
+        # Test the acquired dependency data
+        quilt_native_deps = 0
+        quilt_native_rdeps = 0
+        quilt_deps = 0
+        quilt_rdeps = 0
+        for i,task_obj in enumerate(dep.depends_model):
+            if TYPE_DEP == task_obj[0]:
+                task = task_obj[1]
+                if task.startswith('quilt-native'):
+                    quilt_native_deps += 1
+                elif task.startswith('quilt'):
+                    quilt_deps += 1
+            elif TYPE_RDEP == task_obj[0]:
+                task = task_obj[1]
+                if task.startswith('quilt-native'):
+                    quilt_native_rdeps += 1
+                elif task.startswith('quilt'):
+                    quilt_rdeps += 1
+        # Print results
+        failed = False
+        if 0 < len(dep.depends_model):
+             print(f"Pass:Bitbake dependency count = {len(dep.depends_model)}")
+        else:
+            failed = True
+            print(f"FAIL:Bitbake dependency count = 0")
+        if quilt_native_deps:
+             print(f"Pass:Quilt-native depends count = {quilt_native_deps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt-native depends count = 0")
+        if quilt_native_rdeps:
+             print(f"Pass:Quilt-native rdepends count = {quilt_native_rdeps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt-native rdepends count = 0")
+        if quilt_deps:
+             print(f"Pass:Quilt depends count = {quilt_deps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt depends count = 0")
+        if quilt_rdeps:
+             print(f"Pass:Quilt rdepends count = {quilt_rdeps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt rdepends count = 0")
+        print("* UNIT TEST:STOP")
+        return failed
+
+    # Help method to dynamically test parent window too small
+    def check_screen_size(dep, active_package):
+        global screen_too_small
+        rows, cols = screen.getmaxyx()
+        if (rows >= SCREEN_ROW_MIN) and (cols >= SCREEN_COL_MIN):
+            if screen_too_small:
+                # Now big enough, remove error message and redraw screen
+                dep.draw_frames()
+                active_package.cursor_on(True)
+                screen_too_small = False
+            return True
+        # Test on App init
+        if not dep:
+            # Do not start this app if screen not big enough
+            curses.endwin()
+            print("")
+            print("ERROR(Taskexp_cli): Mininal screen size is %dx%d" % (SCREEN_COL_MIN,SCREEN_ROW_MIN))
+            print("Current screen is Cols=%s,Rows=%d" % (cols,rows))
+            return False
+        # First time window too small
+        if not screen_too_small:
+            active_package.cursor_on(False)
+            dep.screen.addstr(0,2,'[BIGGER WINDOW PLEASE]', curses.color_pair(CURSES_WARNING) | curses.A_BLINK)
+            screen_too_small = True
+        return False
+
+    # Helper method to turn off curses mode
+    def curses_off(screen):
+        if not screen: return
+        # Safe error exit
+        screen.keypad(False)
+        curses.echo()
+        curses.curs_set(1)
+        curses.endwin()
+
+        if unit_test_results:
+            print('\nUnit Test Results:')
+            for line in unit_test_results:
+                print(" %s" % line)
+
+    #
+    # Initialize the ncurse environment
+    #
+
+    screen = curses.initscr()
+    try:
+        if not check_screen_size(None, None):
+            exit(1)
+        try:
+            curses.start_color()
+            curses.use_default_colors();
+            curses.init_pair(0xFF, curses.COLOR_BLACK, curses.COLOR_WHITE);
+            curses.init_pair(CURSES_NORMAL, curses.COLOR_WHITE, curses.COLOR_BLACK)
+            curses.init_pair(CURSES_HIGHLIGHT, curses.COLOR_WHITE, curses.COLOR_BLUE)
+            curses.init_pair(CURSES_WARNING, curses.COLOR_WHITE, curses.COLOR_RED)
+        except:
+            curses.endwin()
+            print("")
+            print("ERROR(Taskexp_cli): Requires 256 colors. Please use this or the equivalent:")
+            print("  $ export TERM='xterm-256color'")
+            exit(1)
+
+        screen.keypad(True)
+        curses.noecho()
+        curses.curs_set(0)
+        screen.refresh();
+    except Exception as e:
+        # Safe error exit
+        curses_off(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+        exit(1)
+
+    try:
+        #
+        # Instantiate the presentation layers
+        #
+
+        dep = DepExplorer(screen)
+
+        #
+        # Prepare bitbake
+        #
+
+        # Fetch bitbake dependecy data
+        ret,cmdline = bitbake_load(server, eventHandler, params, dep, curses_off, screen)
+        if ret: return ret
+
+        #
+        # Preset the views
+        #
+
+        # Cmdline example = ['generateDotGraph', ['acl', 'zlib'], 'build']
+        primary_packages = cmdline[1]
+        dep.package_view.set_primary(primary_packages)
+        dep.dep_view.set_primary(primary_packages)
+        dep.reverse_view.set_primary(primary_packages)
+        dep.help_box_view.set_primary(primary_packages)
+        dep.help_bar_view.show_help(True)
+        active_package = dep.package_view
+        active_package.cursor_on(True)
+        dep.select(primary_packages[0]+'.')
+        if unit_test:
+            alert('UNIT_TEST',screen)
+
+        # Help method to start/stop the filter feature
+        def filter_mode(new_filter_status):
+            global is_filter
+            if is_filter == new_filter_status:
+                # Ignore no changes
+                return
+            if not new_filter_status:
+                # Turn off
+                curses.curs_set(0)
+                #active_package.cursor_on(False)
+                active_package = dep.package_view
+                active_package.cursor_on(True)
+                is_filter = False
+                dep.help_bar_view.show_help(True)
+                dep.filter_str = ''
+                dep.select('')
+            else:
+                # Turn on
+                curses.curs_set(1)
+                dep.help_bar_view.show_help(False)
+                dep.filter_view.clear()
+                dep.filter_view.show(True)
+                dep.filter_view.show_prompt()
+                is_filter = True
+
+        #
+        # Main user loop
+        #
+
+        while not quit:
+            if is_filter:
+                dep.filter_view.show_prompt()
+            if unit_test:
+                c = unit_test_action(active_package)
+            else:
+                c = screen.getch()
+            ch = chr(c)
+
+            # Do not draw if window now too small
+            if not check_screen_size(dep,active_package):
+                continue
+
+            if verbose:
+                if c == CHAR_RETURN:
+                    screen.addstr(0, 4, "|%3d,CR |" % (c))
+                else:
+                    screen.addstr(0, 4, "|%3d,%3s|" % (c,chr(c)))
+
+            # pre-map alternate filter close keys
+            if is_filter and (c == CHAR_ESCAPE):
+                # Alternate exit from filter
+                ch = '/'
+                c = ord(ch)
+
+            # Filter and non-filter mode command keys
+            # https://docs.python.org/3/library/curses.html
+            if c in (curses.KEY_UP,CHAR_UP):
+                active_package.line_up()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif c in (curses.KEY_DOWN,CHAR_DOWN):
+                active_package.line_down()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif curses.KEY_PPAGE == c:
+                active_package.page_up()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif curses.KEY_NPAGE == c:
+                active_package.page_down()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif CHAR_TAB == c:
+                # Tab between boxes
+                active_package.cursor_on(False)
+                if active_package == dep.package_view:
+                    active_package = dep.dep_view
+                elif active_package == dep.dep_view:
+                    active_package = dep.reverse_view
+                else:
+                    active_package = dep.package_view
+                active_package.cursor_on(True)
+            elif curses.KEY_BTAB == c:
+                # Shift-Tab reverse between boxes
+                active_package.cursor_on(False)
+                if active_package == dep.package_view:
+                    active_package = dep.reverse_view
+                elif active_package == dep.reverse_view:
+                    active_package = dep.dep_view
+                else:
+                    active_package = dep.package_view
+                active_package.cursor_on(True)
+            elif (CHAR_RETURN == c):
+                # CR to select
+                selected = active_package.get_selected()
+                if selected:
+                    active_package.cursor_on(False)
+                    active_package = dep.package_view
+                    filter_mode(False)
+                    dep.select(selected)
+                else:
+                    filter_mode(False)
+                    dep.select(primary_packages[0]+'.')
+
+            elif '/' == ch: # Enter/exit dep.filter_view
+                if is_filter:
+                    filter_mode(False)
+                else:
+                    filter_mode(True)
+            elif is_filter:
+                # If in filter mode, re-direct all these other keys to the filter box
+                result = dep.filter_view.input(c,ch)
+                dep.filter_str = dep.filter_view.filter_str
+                dep.select('')
+
+            # Non-filter mode command keys
+            elif 'p' == ch:
+                dep.print_deps(whole_group=False)
+            elif 'P' == ch:
+                dep.print_deps(whole_group=True)
+            elif 'w' == ch:
+                # Toggle the print model
+                if print_model == PRINT_MODEL_1:
+                    print_model = PRINT_MODEL_2
+                else:
+                    print_model = PRINT_MODEL_1
+            elif 's' == ch:
+                # Toggle the sort model
+                if sort_model == SORT_DEPS:
+                    sort_model = SORT_ALPHA
+                elif sort_model == SORT_ALPHA:
+                    if SORT_BITBAKE_ENABLE:
+                        sort_model = TASK_SORT_BITBAKE
+                    else:
+                        sort_model = SORT_DEPS
+                else:
+                    sort_model = SORT_DEPS
+                active_package.cursor_on(False)
+                current_task = active_package.get_selected()
+                dep.package_view.sort()
+                dep.dep_view.sort()
+                dep.reverse_view.sort()
+                active_package = dep.package_view
+                active_package.cursor_on(True)
+                dep.select(current_task)
+                # Announce the new sort model
+                alert("SORT=%s" % ("ALPHA" if (sort_model == SORT_ALPHA) else "DEPS"),screen)
+                alert('',screen)
+
+            elif 'q' == ch:
+                quit = True
+            elif ch in ('h','?'):
+                dep.help_box_view.show_help(True)
+                dep.select(active_package.get_selected())
+
+            #
+            # Debugging commands
+            #
+
+            elif 'V' == ch:
+                verbose = not verbose
+                alert('Verbose=%s' % str(verbose),screen)
+                alert('',screen)
+            elif 'R' == ch:
+                screen.refresh()
+            elif 'B' == ch:
+                # Progress bar unit test
+                dep.progress_view.progress('Test',0,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',10,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',20,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',30,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',40,40)
+                curses.napms(1000)
+                dep.progress_view.clear()
+                dep.help_bar_view.show_help(True)
+            elif 'Q' == ch:
+                # Simulated error
+                curses_off(screen)
+                print('ERROR: simulated error exit')
+                return 1
+
+        # Safe exit
+        curses_off(screen)
+    except Exception as e:
+        # Safe exit on error
+        curses_off(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+
+    # Reminder to pick up your printed results
+    if is_printed:
+        print("")
+        print("You have output ready!")
+        print("  * Your printed dependency file is: %s" % print_file_name)
+        print("  * Your previous results  saved in: %s" % print_file_backup_name)
+        print("")
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index ec5bd4f105..6bd21f1844 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -385,7 +385,7 @@ def main(server, eventHandler, params):
                     main.shutdown = 1
 
                 logger.info("ToasterUI build done, brbe: %s", brbe)
-                continue
+                break
 
             if isinstance(event, (bb.command.CommandCompleted,
                                   bb.command.CommandFailed,
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index 8607d0523b..c2f830d530 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -44,7 +44,7 @@ class BBUIEventQueue:
         for count_tries in range(5):
             ret = self.BBServer.registerEventHandler(self.host, self.port)
 
-            if isinstance(ret, collections.Iterable):
+            if isinstance(ret, collections.abc.Iterable):
                 self.EventHandle, error = ret
             else:
                 self.EventHandle = ret
@@ -65,35 +65,27 @@ class BBUIEventQueue:
         self.server = server
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
+        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-
-        self.eventQueueLock.acquire()
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueNotify.clear()
-
-        self.eventQueueLock.release()
-        return item
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            if not self.eventQueue:
+                return None
+            item = self.eventQueue.pop(0)
+            if not self.eventQueue:
+                self.eventQueueNotify.clear()
+            return item
 
     def waitEvent(self, delay):
         self.eventQueueNotify.wait(delay)
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
diff --git a/bitbake/lib/bb/ui/uihelper.py b/bitbake/lib/bb/ui/uihelper.py
index 48d808ae28..82913e0da8 100644
--- a/bitbake/lib/bb/ui/uihelper.py
+++ b/bitbake/lib/bb/ui/uihelper.py
@@ -49,9 +49,11 @@ class BBUIHelper:
             tid = event._fn + ":" + event._task
             removetid(event.pid, tid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
-        elif isinstance(event, bb.runqueue.runQueueTaskStarted):
-            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed
             self.tasknumber_total = event.stats.total
+            self.setscene_current = event.stats.setscene_active + event.stats.setscene_covered + event.stats.setscene_notcovered
+            self.setscene_total = event.stats.setscene_total
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskProgress):
             if event.pid > 0 and event.pid in self.pidmap:
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index b282d09abf..ebee65d3dd 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -13,10 +13,12 @@ import errno
 import logging
 import bb
 import bb.msg
+import locale
 import multiprocessing
 import fcntl
 import importlib
-from importlib import machinery
+import importlib.machinery
+import importlib.util
 import itertools
 import subprocess
 import glob
@@ -26,6 +28,11 @@ import errno
 import signal
 import collections
 import copy
+import ctypes
+import random
+import socket
+import struct
+import tempfile
 from subprocess import getstatusoutput
 from contextlib import contextmanager
 from ctypes import cdll
@@ -43,7 +50,7 @@ def clean_context():
 
 def get_context():
     return _context
-    
+
 
 def set_context(ctx):
     _context = ctx
@@ -205,8 +212,8 @@ def explode_dep_versions2(s, *, sort=True):
             inversion = True
             # This list is based on behavior and supported comparisons from deb, opkg and rpm.
             #
-            # Even though =<, <<, ==, !=, =>, and >> may not be supported, 
-            # we list each possibly valid item. 
+            # Even though =<, <<, ==, !=, =>, and >> may not be supported,
+            # we list each possibly valid item.
             # The build system is responsible for validation of what it supports.
             if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
                 lastcmp = i[0:2]
@@ -251,7 +258,7 @@ def explode_dep_versions(s):
     """
     Take an RDEPENDS style string of format:
     "DEPEND1 (optional version) DEPEND2 (optional version) ..."
-    skip null value and items appeared in dependancy string multiple times
+    skip null value and items appeared in dependency string multiple times
     and return a dictionary of dependencies and versions.
     """
     r = explode_dep_versions2(s)
@@ -340,7 +347,7 @@ def _print_exception(t, value, tb, realfile, text, context):
         exception = traceback.format_exception_only(t, value)
         error.append('Error executing a python function in %s:\n' % realfile)
 
-        # Strip 'us' from the stack (better_exec call) unless that was where the 
+        # Strip 'us' from the stack (better_exec call) unless that was where the
         # error came from
         if tb.tb_next is not None:
             tb = tb.tb_next
@@ -379,7 +386,7 @@ def _print_exception(t, value, tb, realfile, text, context):
 
         error.append("Exception: %s" % ''.join(exception))
 
-        # If the exception is from spwaning a task, let's be helpful and display
+        # If the exception is from spawning a task, let's be helpful and display
         # the output (which hopefully includes stderr).
         if isinstance(value, subprocess.CalledProcessError) and value.output:
             error.append("Subprocess output:")
@@ -400,7 +407,7 @@ def better_exec(code, context, text = None, realfile = "<code>", pythonexception
         code = better_compile(code, realfile, realfile)
     try:
         exec(code, get_context(), context)
-    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError):
+    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError, bb.process.ExecutionError):
         # Error already shown so passthrough, no need for traceback
         raise
     except Exception as e:
@@ -427,12 +434,14 @@ def better_eval(source, locals, extraglobals = None):
     return eval(source, ctx, locals)
 
 @contextmanager
-def fileslocked(files):
+def fileslocked(files, *args, **kwargs):
     """Context manager for locking and unlocking file locks."""
     locks = []
     if files:
         for lockfile in files:
-            locks.append(bb.utils.lockfile(lockfile))
+            l = bb.utils.lockfile(lockfile, *args, **kwargs)
+            if l is not None:
+                locks.append(l)
 
     try:
         yield
@@ -451,9 +460,16 @@ def lockfile(name, shared=False, retry=True, block=False):
     consider the possibility of sending a signal to the process to break
     out - at which point you want block=True rather than retry=True.
     """
+    basename = os.path.basename(name)
+    if len(basename) > 255:
+        root, ext = os.path.splitext(basename)
+        basename = root[:255 - len(ext)] + ext
+
     dirname = os.path.dirname(name)
     mkdirhier(dirname)
 
+    name = os.path.join(dirname, basename)
+
     if not os.access(dirname, os.W_OK):
         logger.error("Unable to acquire lock '%s', directory is not writable",
                      name)
@@ -487,7 +503,7 @@ def lockfile(name, shared=False, retry=True, block=False):
                     return lf
             lf.close()
         except OSError as e:
-            if e.errno == errno.EACCES:
+            if e.errno == errno.EACCES or e.errno == errno.ENAMETOOLONG:
                 logger.error("Unable to acquire lock '%s', %s",
                              e.strerror, name)
                 sys.exit(1)
@@ -532,7 +548,12 @@ def md5_file(filename):
     Return the hex string representation of the MD5 checksum of filename.
     """
     import hashlib
-    return _hasher(hashlib.md5(), filename)
+    try:
+        sig = hashlib.new('MD5', usedforsecurity=False)
+    except TypeError:
+        # Some configurations don't appear to support two arguments
+        sig = hashlib.new('MD5')
+    return _hasher(sig, filename)
 
 def sha256_file(filename):
     """
@@ -583,11 +604,25 @@ def preserved_envvars():
     v = [
         'BBPATH',
         'BB_PRESERVE_ENV',
-        'BB_ENV_WHITELIST',
-        'BB_ENV_EXTRAWHITE',
+        'BB_ENV_PASSTHROUGH_ADDITIONS',
     ]
     return v + preserved_envvars_exported()
 
+def check_system_locale():
+    """Make sure the required system locale are available and configured"""
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
@@ -615,21 +650,21 @@ def filter_environment(good_vars):
 
 def approved_variables():
     """
-    Determine and return the list of whitelisted variables which are approved
+    Determine and return the list of variables which are approved
     to remain in the environment.
     """
     if 'BB_PRESERVE_ENV' in os.environ:
         return os.environ.keys()
     approved = []
-    if 'BB_ENV_WHITELIST' in os.environ:
-        approved = os.environ['BB_ENV_WHITELIST'].split()
-        approved.extend(['BB_ENV_WHITELIST'])
+    if 'BB_ENV_PASSTHROUGH' in os.environ:
+        approved = os.environ['BB_ENV_PASSTHROUGH'].split()
+        approved.extend(['BB_ENV_PASSTHROUGH'])
     else:
         approved = preserved_envvars()
-    if 'BB_ENV_EXTRAWHITE' in os.environ:
-        approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
-        if 'BB_ENV_EXTRAWHITE' not in approved:
-            approved.extend(['BB_ENV_EXTRAWHITE'])
+    if 'BB_ENV_PASSTHROUGH_ADDITIONS' in os.environ:
+        approved.extend(os.environ['BB_ENV_PASSTHROUGH_ADDITIONS'].split())
+        if 'BB_ENV_PASSTHROUGH_ADDITIONS' not in approved:
+            approved.extend(['BB_ENV_PASSTHROUGH_ADDITIONS'])
     return approved
 
 def clean_environment():
@@ -683,8 +718,8 @@ def remove(path, recurse=False, ionice=False):
         return
     if recurse:
         for name in glob.glob(path):
-            if _check_unsafe_delete_path(path):
-                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
+            if _check_unsafe_delete_path(name):
+                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % name)
         # shutil.rmtree(name) would be ideal but its too slow
         cmd = []
         if ionice:
@@ -710,9 +745,9 @@ def prunedir(topdir, ionice=False):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    """ 
+    """
     See if var ends with any of the suffixes listed and
-    remove it if found 
+    remove it if found
     """
     for suffix in suffixes:
         if suffix and var.endswith(suffix):
@@ -723,7 +758,8 @@ def mkdirhier(directory):
     """Create a directory like 'mkdir -p', but does not complain if
     directory already exists like os.makedirs
     """
-
+    if '${' in str(directory):
+        bb.fatal("Directory name {} contains unexpanded bitbake variable. This may cause build failures and WORKDIR polution.".format(directory))
     try:
         os.makedirs(directory)
     except OSError as e:
@@ -742,7 +778,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if not sstat:
             sstat = os.lstat(src)
     except Exception as e:
-        print("movefile: Stating source file failed...", e)
+        logger.warning("movefile: Stating source file failed...", e)
         return None
 
     destexists = 1
@@ -770,7 +806,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
             os.unlink(src)
             return os.lstat(dest)
         except Exception as e:
-            print("movefile: failed to properly create symlink:", dest, "->", target, e)
+            logger.warning("movefile: failed to properly create symlink:", dest, "->", target, e)
             return None
 
     renamefailed = 1
@@ -782,12 +818,12 @@ def movefile(src, dest, newmtime = None, sstat = None):
 
     if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
         try:
-            os.rename(src, destpath)
+            bb.utils.rename(src, destpath)
             renamefailed = 0
         except Exception as e:
             if e.errno != errno.EXDEV:
                 # Some random error.
-                print("movefile: Failed to move", src, "to", dest, e)
+                logger.warning("movefile: Failed to move", src, "to", dest, e)
                 return None
             # Invalid cross-device-link 'bind' mounted or actually Cross-Device
 
@@ -796,16 +832,16 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if stat.S_ISREG(sstat[stat.ST_MODE]):
             try: # For safety copy then move it over.
                 shutil.copyfile(src, destpath + "#new")
-                os.rename(destpath + "#new", destpath)
+                bb.utils.rename(destpath + "#new", destpath)
                 didcopy = 1
             except Exception as e:
-                print('movefile: copy', src, '->', dest, 'failed.', e)
+                logger.warning('movefile: copy', src, '->', dest, 'failed.', e)
                 return None
         else:
             #we don't yet handle special, so we need to fall back to /bin/mv
             a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
             if a[0] != 0:
-                print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
+                logger.warning("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
                 return None # failure
         try:
             if didcopy:
@@ -813,7 +849,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
                 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
                 os.unlink(src)
         except Exception as e:
-            print("movefile: Failed to chown/chmod/unlink", dest, e)
+            logger.warning("movefile: Failed to chown/chmod/unlink", dest, e)
             return None
 
     if newmtime:
@@ -874,7 +910,7 @@ def copyfile(src, dest, newmtime = None, sstat = None):
 
             # For safety copy then move it over.
             shutil.copyfile(src, dest + "#new")
-            os.rename(dest + "#new", dest)
+            bb.utils.rename(dest + "#new", dest)
         except Exception as e:
             logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
             return False
@@ -965,13 +1001,16 @@ def umask(new_mask):
         os.umask(current_mask)
 
 def to_boolean(string, default=None):
-    """ 
+    """
     Check input string and return boolean value True/False/None
-    depending upon the checks 
+    depending upon the checks
     """
     if not string:
         return default
 
+    if isinstance(string, int):
+        return string != 0
+
     normalized = string.lower()
     if normalized in ("y", "yes", "1", "true"):
         return True
@@ -1103,7 +1142,10 @@ def get_referenced_vars(start_expr, d):
 
 
 def cpu_count():
-    return multiprocessing.cpu_count()
+    try:
+        return len(os.sched_getaffinity(0))
+    except OSError:
+        return multiprocessing.cpu_count()
 
 def nonblockingfd(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
@@ -1178,7 +1220,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
         variables: a list of variable names to look for. Functions
             may also be specified, but must be specified with '()' at
             the end of the name. Note that the function doesn't have
-            any intrinsic understanding of _append, _prepend, _remove,
+            any intrinsic understanding of :append, :prepend, :remove,
             or overrides, so these are considered as part of the name.
             These values go into a regular expression, so regular
             expression syntax is allowed.
@@ -1590,33 +1632,89 @@ def set_process_name(name):
     except:
         pass
 
+def enable_loopback_networking():
+    # From bits/ioctls.h
+    SIOCGIFFLAGS = 0x8913
+    SIOCSIFFLAGS = 0x8914
+    SIOCSIFADDR = 0x8916
+    SIOCSIFNETMASK = 0x891C
+
+    # if.h
+    IFF_UP = 0x1
+    IFF_RUNNING = 0x40
+
+    # bits/socket.h
+    AF_INET = 2
+
+    # char ifr_name[IFNAMSIZ=16]
+    ifr_name = struct.pack("@16s", b"lo")
+    def netdev_req(fd, req, data = b""):
+        # Pad and add interface name
+        data = ifr_name + data + (b'\x00' * (16 - len(data)))
+        # Return all data after interface name
+        return fcntl.ioctl(fd, req, data)[16:]
+
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_IP) as sock:
+        fd = sock.fileno()
+
+        # struct sockaddr_in ifr_addr { unsigned short family; uint16_t sin_port ; uint32_t in_addr; }
+        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 127, 0, 0, 1)
+        netdev_req(fd, SIOCSIFADDR, req)
+
+        # short ifr_flags
+        flags = struct.unpack_from('@h', netdev_req(fd, SIOCGIFFLAGS))[0]
+        flags |= IFF_UP | IFF_RUNNING
+        netdev_req(fd, SIOCSIFFLAGS, struct.pack('@h', flags))
+
+        # struct sockaddr_in ifr_netmask
+        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 255, 0, 0, 0)
+        netdev_req(fd, SIOCSIFNETMASK, req)
+
+def disable_network(uid=None, gid=None):
+    """
+    Disable networking in the current process if the kernel supports it, else
+    just return after logging to debug. To do this we need to create a new user
+    namespace, then map back to the original uid/gid.
+    """
+    libc = ctypes.CDLL('libc.so.6')
+
+    # From sched.h
+    # New user namespace
+    CLONE_NEWUSER = 0x10000000
+    # New network namespace
+    CLONE_NEWNET = 0x40000000
+
+    if uid is None:
+        uid = os.getuid()
+    if gid is None:
+        gid = os.getgid()
+
+    ret = libc.unshare(CLONE_NEWNET | CLONE_NEWUSER)
+    if ret != 0:
+        logger.debug("System doesn't support disabling network without admin privs")
+        return
+    with open("/proc/self/uid_map", "w") as f:
+        f.write("%s %s 1" % (uid, uid))
+    with open("/proc/self/setgroups", "w") as f:
+        f.write("deny")
+    with open("/proc/self/gid_map", "w") as f:
+        f.write("%s %s 1" % (gid, gid))
+
 def export_proxies(d):
+    from bb.fetch2 import get_fetcher_environment
     """ export common proxies variables from datastore to environment """
-    import os
-
-    variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
-                    'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
-                    'GIT_PROXY_COMMAND']
-    exported = False
-
-    for v in variables:
-        if v in os.environ.keys():
-            exported = True
-        else:
-            v_proxy = d.getVar(v)
-            if v_proxy is not None:
-                os.environ[v] = v_proxy
-                exported = True
-
-    return exported
-
+    newenv = get_fetcher_environment(d)
+    for v in newenv:
+        os.environ[v] = newenv[v]
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
         logger.debug('Loading plugin %s' % name)
         spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
         if spec:
-            return spec.loader.load_module()
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
 
     logger.debug('Loading plugins from %s...' % pluginpath)
 
@@ -1669,3 +1767,102 @@ def is_semver(version):
         return False
 
     return True
+
+# Wrapper around os.rename which can handle cross device problems
+# e.g. from container filesystems
+def rename(src, dst):
+    try:
+        os.rename(src, dst)
+    except OSError as err:
+        if err.errno == 18:
+            # Invalid cross-device link error
+            shutil.move(src, dst)
+        else:
+            raise err
+
+@contextmanager
+def environment(**envvars):
+    """
+    Context manager to selectively update the environment with the specified mapping.
+    """
+    backup = dict(os.environ)
+    try:
+        os.environ.update(envvars)
+        yield
+    finally:
+        for var in envvars:
+            if var in backup:
+                os.environ[var] = backup[var]
+            elif var in os.environ:
+                del os.environ[var]
+
+def is_local_uid(uid=''):
+    """
+    Check whether uid is a local one or not.
+    Can't use pwd module since it gets all UIDs, not local ones only.
+    """
+    if not uid:
+        uid = os.getuid()
+    with open('/etc/passwd', 'r') as f:
+        for line in f:
+            line_split = line.split(':')
+            if len(line_split) < 3:
+                continue
+            if str(uid) == line_split[2]:
+                return True
+    return False
+
+def mkstemp(suffix=None, prefix=None, dir=None, text=False):
+    """
+    Generates a unique filename, independent of time.
+
+    mkstemp() in glibc (at least) generates unique file names based on the
+    current system time. When combined with highly parallel builds, and
+    operating over NFS (e.g. shared sstate/downloads) this can result in
+    conflicts and race conditions.
+
+    This function adds additional entropy to the file name so that a collision
+    is independent of time and thus extremely unlikely.
+    """
+    entropy = "".join(random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890", k=20))
+    if prefix:
+        prefix = prefix + entropy
+    else:
+        prefix = tempfile.gettempprefix() + entropy
+    return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
+
+def path_is_descendant(descendant, ancestor):
+    """
+    Returns True if the path `descendant` is a descendant of `ancestor`
+    (including being equivalent to `ancestor` itself). Otherwise returns False.
+    Correctly accounts for symlinks, bind mounts, etc. by using
+    os.path.samestat() to compare paths
+
+    May raise any exception that os.stat() raises
+    """
+
+    ancestor_stat = os.stat(ancestor)
+
+    # Recurse up each directory component of the descendant to see if it is
+    # equivalent to the ancestor
+    check_dir = os.path.abspath(descendant).rstrip("/")
+    while check_dir:
+        check_stat = os.stat(check_dir)
+        if os.path.samestat(check_stat, ancestor_stat):
+            return True
+        check_dir = os.path.dirname(check_dir).rstrip("/")
+
+    return False
+
+# If we don't have a timeout of some kind and a process/thread exits badly (for example
+# OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
+# we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.
+@contextmanager
+def lock_timeout(lock):
+    held = lock.acquire(timeout=5*60)
+    try:
+        if not held:
+            os._exit(1)
+        yield held
+    finally:
+        lock.release()
diff --git a/bitbake/lib/bb/xattr.py b/bitbake/lib/bb/xattr.py
new file mode 100755
index 0000000000..7b634944a4
--- /dev/null
+++ b/bitbake/lib/bb/xattr.py
@@ -0,0 +1,126 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import ctypes
+import os
+import errno
+
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+fsencoding = sys.getfilesystemencoding()
+
+
+libc.listxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+libc.llistxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+
+
+def listxattr(path, follow=True):
+    func = libc.listxattr if follow else libc.llistxattr
+
+    os_path = os.fsencode(path)
+
+    while True:
+        length = func(os_path, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return []
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return [a.decode(fsencoding) for a in arr.raw.split(b"\x00") if a]
+
+
+libc.getxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+libc.lgetxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+
+
+def getxattr(path, name, follow=True):
+    func = libc.getxattr if follow else libc.lgetxattr
+
+    os_path = os.fsencode(path)
+    os_name = os.fsencode(name)
+
+    while True:
+        length = func(os_path, os_name, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            if err == errno.ENODATA:
+                return None
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return ""
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, os_name, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return arr.raw
+
+
+def get_all_xattr(path, follow=True):
+    attrs = {}
+
+    names = listxattr(path, follow)
+
+    for name in names:
+        value = getxattr(path, name, follow)
+        if value is None:
+            # This can happen if a value is erased after listxattr is called,
+            # so ignore it
+            continue
+        attrs[name] = value
+
+    return attrs
+
+
+def main():
+    import argparse
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    attrs = get_all_xattr(args.path)
+
+    for name, value in attrs.items():
+        try:
+            value = value.decode(fsencoding)
+        except UnicodeDecodeError:
+            pass
+
+        print(f"{name} = {value}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())