19 files changed, 398 insertions, 239 deletions
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index 768cce84e9..574e0de5be 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -9,7 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
-__version__ = "2.7.3"
+__version__ = "2.9.1"
 import sys
 if sys.version_info < (3, 8, 0):
@@ -36,6 +36,7 @@ class BBHandledException(Exception):
 import os
 import logging
+from collections import namedtuple
 class NullHandler(logging.Handler):
@@ -103,26 +104,6 @@ class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin):
        self.setup_bblogger(logger.name)
        super().__init__(logger, *args, **kwargs)
-    if sys.version_info < (3, 6):
-        # These properties were added in Python 3.6. Add them in older versions
-        # for compatibility
-        @property
-        def manager(self):
-            return self.logger.manager
-        @manager.setter
-        def manager(self, value):
-            self.logger.manager = value
-        @property
-        def name(self):
-            return self.logger.name
-        def __repr__(self):
-            logger = self.logger
-            level = logger.getLevelName(logger.getEffectiveLevel())
-            return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
 logging.LoggerAdapter = BBLoggerAdapter
 logger = logging.getLogger("BitBake")
@@ -227,3 +208,14 @@ def deprecate_import(current, modulename, fromlist, renames = None):
        setattr(sys.modules[current], newname, newobj)
+TaskData = namedtuple("TaskData", [
+    "pn",
+    "taskname",
+    "fn",
+    "deps",
+    "provides",
+    "taskhash",
+    "unihash",
+    "hashfn",
+    "taskhash_deps",
+])
diff --git a/bitbake/lib/bb/asyncrpc/__init__.py b/bitbake/lib/bb/asyncrpc/__init__.py
index 639e1607f8..a4371643d7 100644
--- a/bitbake/lib/bb/asyncrpc/__init__.py
+++ b/bitbake/lib/bb/asyncrpc/__init__.py
@@ -5,7 +5,7 @@
 #
-from .client import AsyncClient, Client, ClientPool
+from .client import AsyncClient, Client
 from .serv import AsyncServer, AsyncServerConnection
 from .connection import DEFAULT_MAX_CHUNK
 from .exceptions import (
diff --git a/bitbake/lib/bb/asyncrpc/client.py b/bitbake/lib/bb/asyncrpc/client.py
index 29a5ab76aa..9be49261c0 100644
--- a/bitbake/lib/bb/asyncrpc/client.py
+++ b/bitbake/lib/bb/asyncrpc/client.py
@@ -10,11 +10,41 @@ import json
 import os
 import socket
 import sys
+import re
 import contextlib
 from threading import Thread
 from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK
 from .exceptions import ConnectionClosedError, InvokeError
+UNIX_PREFIX = "unix://"
+WS_PREFIX = "ws://"
+WSS_PREFIX = "wss://"
+ADDR_TYPE_UNIX = 0
+ADDR_TYPE_TCP = 1
+ADDR_TYPE_WS = 2
+WEBSOCKETS_MIN_VERSION = (9, 1)
+# Need websockets 10 with python 3.10+
+if sys.version_info >= (3, 10, 0):
+    WEBSOCKETS_MIN_VERSION = (10, 0)
+def parse_address(addr):
+    if addr.startswith(UNIX_PREFIX):
+        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
+    elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
+        return (ADDR_TYPE_WS, (addr,))
+    else:
+        m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
+        if m is not None:
+            host = m.group("host")
+            port = m.group("port")
+        else:
+            host, port = addr.split(":")
+        return (ADDR_TYPE_TCP, (host, int(port)))
 class AsyncClient(object):
    def __init__(
@@ -63,8 +93,30 @@ class AsyncClient(object):
    async def connect_websocket(self, uri):
        import websockets
+        try:
+            version = tuple(
+                int(v)
+                for v in websockets.__version__.split(".")[
+                    0 : len(WEBSOCKETS_MIN_VERSION)
+                ]
+            )
+        except ValueError:
+            raise ImportError(
+                f"Unable to parse websockets version '{websockets.__version__}'"
+            )
+        if version < WEBSOCKETS_MIN_VERSION:
+            min_ver_str = ".".join(str(v) for v in WEBSOCKETS_MIN_VERSION)
+            raise ImportError(
+                f"Websockets version {websockets.__version__} is less than minimum required version {min_ver_str}"
+            )
        async def connect_sock():
-            websocket = await websockets.connect(uri, ping_interval=None)
+            websocket = await websockets.connect(
+                uri,
+                ping_interval=None,
+                open_timeout=self.timeout,
+            )
            return WebsocketConnection(websocket, self.timeout)
        self._connect_sock = connect_sock
@@ -202,85 +254,9 @@ class Client(object):
    def close(self):
        if self.loop:
            self.loop.run_until_complete(self.client.close())
-            if sys.version_info >= (3, 6):
-                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-            self.loop.close()
-        self.loop = None
-    def __enter__(self):
-        return self
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-        return False
-class ClientPool(object):
-    def __init__(self, max_clients):
-        self.avail_clients = []
-        self.num_clients = 0
-        self.max_clients = max_clients
-        self.loop = None
-        self.client_condition = None
-    @abc.abstractmethod
-    async def _new_client(self):
-        raise NotImplementedError("Must be implemented in derived class")
-    def close(self):
-        if self.client_condition:
-            self.client_condition = None
-        if self.loop:
-            self.loop.run_until_complete(self.__close_clients())
            self.loop.run_until_complete(self.loop.shutdown_asyncgens())
            self.loop.close()
-            self.loop = None
+        self.loop = None
-    def run_tasks(self, tasks):
-        if not self.loop:
-            self.loop = asyncio.new_event_loop()
-        thread = Thread(target=self.__thread_main, args=(tasks,))
-        thread.start()
-        thread.join()
-    @contextlib.asynccontextmanager
-    async def get_client(self):
-        async with self.client_condition:
-            if self.avail_clients:
-                client = self.avail_clients.pop()
-            elif self.num_clients < self.max_clients:
-                self.num_clients += 1
-                client = await self._new_client()
-            else:
-                while not self.avail_clients:
-                    await self.client_condition.wait()
-                client = self.avail_clients.pop()
-        try:
-            yield client
-        finally:
-            async with self.client_condition:
-                self.avail_clients.append(client)
-                self.client_condition.notify()
-    def __thread_main(self, tasks):
-        async def process_task(task):
-            async with self.get_client() as client:
-                await task(client)
-        asyncio.set_event_loop(self.loop)
-        if not self.client_condition:
-            self.client_condition = asyncio.Condition()
-        tasks = [process_task(t) for t in tasks]
-        self.loop.run_until_complete(asyncio.gather(*tasks))
-    async def __close_clients(self):
-        for c in self.avail_clients:
-            await c.close()
-        self.avail_clients = []
-        self.num_clients = 0
    def __enter__(self):
        return self
diff --git a/bitbake/lib/bb/asyncrpc/serv.py b/bitbake/lib/bb/asyncrpc/serv.py
index a66117acad..667217c5c1 100644
--- a/bitbake/lib/bb/asyncrpc/serv.py
+++ b/bitbake/lib/bb/asyncrpc/serv.py
@@ -138,14 +138,20 @@ class StreamServer(object):
 class TCPStreamServer(StreamServer):
-    def __init__(self, host, port, handler, logger):
+    def __init__(self, host, port, handler, logger, *, reuseport=False):
        super().__init__(handler, logger)
        self.host = host
        self.port = port
+        self.reuseport = reuseport
    def start(self, loop):
        self.server = loop.run_until_complete(
-            asyncio.start_server(self.handle_stream_client, self.host, self.port)
+            asyncio.start_server(
+                self.handle_stream_client,
+                self.host,
+                self.port,
+                reuse_port=self.reuseport,
+            )
        )
        for s in self.server.sockets:
@@ -209,11 +215,12 @@ class UnixStreamServer(StreamServer):
 class WebsocketsServer(object):
-    def __init__(self, host, port, handler, logger):
+    def __init__(self, host, port, handler, logger, *, reuseport=False):
        self.host = host
        self.port = port
        self.handler = handler
        self.logger = logger
+        self.reuseport = reuseport
    def start(self, loop):
        import websockets.server
@@ -224,6 +231,7 @@ class WebsocketsServer(object):
                self.host,
                self.port,
                ping_interval=None,
+                reuse_port=self.reuseport,
            )
        )
@@ -262,14 +270,26 @@ class AsyncServer(object):
        self.loop = None
        self.run_tasks = []
-    def start_tcp_server(self, host, port):
+    def start_tcp_server(self, host, port, *, reuseport=False):
-        self.server = TCPStreamServer(host, port, self._client_handler, self.logger)
+        self.server = TCPStreamServer(
+            host,
+            port,
+            self._client_handler,
+            self.logger,
+            reuseport=reuseport,
+        )
    def start_unix_server(self, path):
        self.server = UnixStreamServer(path, self._client_handler, self.logger)
-    def start_websocket_server(self, host, port):
+    def start_websocket_server(self, host, port, reuseport=False):
-        self.server = WebsocketsServer(host, port, self._client_handler, self.logger)
+        self.server = WebsocketsServer(
+            host,
+            port,
+            self._client_handler,
+            self.logger,
+            reuseport=reuseport,
+        )
    async def _client_handler(self, socket):
        address = socket.address
@@ -368,8 +388,7 @@ class AsyncServer(object):
            self._serve_forever(tasks)
-            if sys.version_info >= (3, 6):
+            self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
            self.loop.close()
        queue = multiprocessing.Queue()
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 44d08f5c55..ab8bce3d57 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -197,6 +197,8 @@ def exec_func(func, d, dirs = None):
        for cdir in d.expand(cleandirs).split():
            bb.utils.remove(cdir, True)
            bb.utils.mkdirhier(cdir)
+            if cdir == oldcwd:
+                os.chdir(cdir)
    if flags and dirs is None:
        dirs = flags.get('dirs')
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 2e8b7ced3c..691bdff75e 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -484,19 +484,34 @@ class ShellParser():
        """
        words = list(words)
-        for word in list(words):
+        for word in words:
            wtree = pyshlex.make_wordtree(word[1])
            for part in wtree:
                if not isinstance(part, list):
                    continue
-                if part[0] in ('`', '$('):
+                candidates = [part]
-                    command = pyshlex.wordtree_as_string(part[1:-1])
-                    self._parse_shell(command)
+                # If command is of type:
+                #
-                    if word[0] in ("cmd_name", "cmd_word"):
+                #   var="... $(cmd [...]) ..."
-                        if word in words:
+                #
-                            words.remove(word)
+                # Then iterate on what's between the quotes and if we find a
+                # list, make that what we check for below.
+                if len(part) >= 3 and part[0] == '"':
+                    for p in part[1:-1]:
+                        if isinstance(p, list):
+                            candidates.append(p)
+                for candidate in candidates:
+                    if len(candidate) >= 2:
+                        if candidate[0] in ('`', '$('):
+                            command = pyshlex.wordtree_as_string(candidate[1:-1])
+                            self._parse_shell(command)
+                            if word[0] in ("cmd_name", "cmd_word"):
+                                if word in words:
+                                    words.remove(word)
        usetoken = False
        for word in words:
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index c5bfef55d6..6754f986bf 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -315,13 +315,13 @@ class BBCooker:
                dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
                upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
                if upstream:
-                    import socket
                    try:
-                        sock = socket.create_connection(upstream.split(":"), 5)
+                        with hashserv.create_client(upstream) as client:
-                        sock.close()
+                            client.ping()
-                    except socket.error as e:
+                    except (ConnectionError, ImportError) as e:
                        bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
                                 % (upstream, repr(e)))
+                        upstream = None
                self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
                self.hashserv = hashserv.create_server(
@@ -680,14 +680,14 @@ class BBCooker:
        bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
        return taskdata, runlist
-    def prepareTreeData(self, pkgs_to_build, task):
+    def prepareTreeData(self, pkgs_to_build, task, halt=False):
        """
        Prepare a runqueue and taskdata object for iteration over pkgs_to_build
        """
        # We set halt to False here to prevent unbuildable targets raising
        # an exception when we're just generating data
-        taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
+        taskdata, runlist = self.buildTaskData(pkgs_to_build, task, halt, allowincomplete=True)
        return runlist, taskdata
@@ -701,7 +701,7 @@ class BBCooker:
        if not task.startswith("do_"):
            task = "do_%s" % task
-        runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
+        runlist, taskdata = self.prepareTreeData(pkgs_to_build, task, halt=True)
        rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
        rq.rqdata.prepare()
        return self.buildDependTree(rq, taskdata)
@@ -1459,7 +1459,6 @@ class BBCooker:
                    if t in task or getAllTaskSignatures:
                        try:
-                            rq.rqdata.prepare_task_hash(tid)
                            sig.append([pn, t, rq.rqdata.get_task_unihash(tid)])
                        except KeyError:
                            sig.append(self.getTaskSignatures(target, [t])[0])
@@ -1813,8 +1812,8 @@ class CookerCollectFiles(object):
            bb.event.fire(CookerExit(), eventdata)
        # We need to track where we look so that we can know when the cache is invalid. There
-        # is no nice way to do this, this is horrid. We intercept the os.listdir()
+        # is no nice way to do this, this is horrid. We intercept the os.listdir() and os.scandir()
-        # (or os.scandir() for python 3.6+) calls while we run glob().
+        # calls while we run glob().
        origlistdir = os.listdir
        if hasattr(os, 'scandir'):
            origscandir = os.scandir
@@ -2225,9 +2224,8 @@ class CookerParser(object):
        for process in self.processes:
            process.join()
-            # Added in 3.7, cleans up zombies
+            # clean up zombies
-            if hasattr(process, "close"):
+            process.close()
-                process.close()
        bb.codeparser.parser_cache_save()
        bb.codeparser.parser_cache_savemerge()
@@ -2237,12 +2235,13 @@ class CookerParser(object):
            profiles = []
            for i in self.process_names:
                logfile = "profile-parse-%s.log" % i
-                if os.path.exists(logfile):
+                if os.path.exists(logfile) and os.path.getsize(logfile):
                    profiles.append(logfile)
-            pout = "profile-parse.log.processed"
+            if profiles:
-            bb.utils.process_profilelog(profiles, pout = pout)
+                pout = "profile-parse.log.processed"
-            print("Processed parsing statistics saved to %s" % (pout))
+                bb.utils.process_profilelog(profiles, pout = pout)
+                print("Processed parsing statistics saved to %s" % (pout))
    def final_cleanup(self):
        if self.syncthread:
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
index 01d49435c3..e611736f06 100644
--- a/bitbake/lib/bb/fetch2/crate.py
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -70,6 +70,7 @@ class Crate(Wget):
            host = 'crates.io/api/v1/crates'
        ud.url = "https://%s/%s/%s/download" % (host, name, version)
+        ud.versionsurl = "https://%s/%s/versions" % (host, name)
        ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
        if 'name' not in ud.parm:
            ud.parm['name'] = '%s-%s' % (name, version)
@@ -139,3 +140,11 @@ class Crate(Wget):
            mdpath = os.path.join(bbpath, cratepath, mdfile)
            with open(mdpath, "w") as f:
                json.dump(metadata, f)
+    def latest_versionstring(self, ud, d):
+        from functools import cmp_to_key
+        json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
+        versions = [(0, i["num"], "") for i in json_data["versions"]]
+        versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
+        return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
index f40ce2eaa5..eb3e0c6a6b 100644
--- a/bitbake/lib/bb/fetch2/gcp.py
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -23,6 +23,7 @@ import urllib.parse, urllib.error
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
 class GCP(FetchMethod):
    """
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index ff5f8dc755..b55e885d7b 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -268,7 +268,7 @@ class NpmShrinkWrap(FetchMethod):
    def unpack(self, ud, rootdir, d):
        """Unpack the downloaded dependencies"""
-        destdir = d.getVar("S")
+        destdir = rootdir
        destsuffix = ud.parm.get("destsuffix")
        if destsuffix:
            destdir = os.path.join(rootdir, destsuffix)
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index d40e4d2909..0852108e7d 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -210,3 +210,6 @@ class Svn(FetchMethod):
    def _build_revision(self, ud, d):
        return ud.revision
+    def supports_checksum(self, urldata):
+        return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index dc025800e6..2e92117634 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -87,7 +87,10 @@ class Wget(FetchMethod):
        if not ud.localfile:
            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+        if ud.type == 'ftp' or ud.type == 'ftps':
+            self.basecmd += " --passive-ftp"
        if not self.check_certs(d):
            self.basecmd += " --no-check-certificate"
@@ -105,7 +108,8 @@ class Wget(FetchMethod):
        fetchcmd = self.basecmd
-        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
+        dldir = os.path.realpath(d.getVar("DL_DIR"))
+        localpath = os.path.join(dldir, ud.localfile) + ".tmp"
        bb.utils.mkdirhier(os.path.dirname(localpath))
        fetchcmd += " -O %s" % shlex.quote(localpath)
@@ -125,12 +129,21 @@ class Wget(FetchMethod):
        uri = ud.url.split(";")[0]
        if os.path.exists(ud.localpath):
            # file exists, but we didnt complete it.. trying again..
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
+            fetchcmd += " -c -P " + dldir + " '" + uri + "'"
        else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
+            fetchcmd += " -P " + dldir + " '" + uri + "'"
        self._runwget(ud, d, fetchcmd, False)
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
+        if os.path.getsize(localpath) == 0:
+            os.remove(localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
        # original file, which might be a race (imagine two recipes referencing the same
        # source, one with an incorrect checksum)
@@ -140,15 +153,6 @@ class Wget(FetchMethod):
        # Our lock prevents multiple writers but mirroring code may grab incomplete files
        os.rename(localpath, localpath[:-4])
-        # Sanity check since wget can pretend it succeed when it didn't
-        # Also, this used to happen if sourceforge sent us to the mirror page
-        if not os.path.exists(ud.localpath):
-            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
-        if os.path.getsize(ud.localpath) == 0:
-            os.remove(ud.localpath)
-            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
        return True
    def checkstatus(self, fetch, ud, d, try_again=True):
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index a4358f1374..7ffdaa6fd7 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -49,20 +49,23 @@ class SkipPackage(SkipRecipe):
 __mtime_cache = {}
 def cached_mtime(f):
    if f not in __mtime_cache:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
    return __mtime_cache[f]
 def cached_mtime_noerror(f):
    if f not in __mtime_cache:
        try:
-            __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+            res = os.stat(f)
+            __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
        except OSError:
            return 0
    return __mtime_cache[f]
 def check_mtime(f, mtime):
    try:
-        current_mtime = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        current_mtime = (res.st_mtime_ns, res.st_size, res.st_ino)
        __mtime_cache[f] = current_mtime
    except OSError:
        current_mtime = 0
@@ -70,7 +73,8 @@ def check_mtime(f, mtime):
 def update_mtime(f):
    try:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
    except OSError:
        if f in __mtime_cache:
            del __mtime_cache[f]
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index cd1c998f8f..c13e4b9755 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -34,6 +34,7 @@ __infunc__ = []
 __inpython__ = False
 __body__   = []
 __classname__ = ""
+__residue__ = []
 cached_statements = {}
@@ -80,7 +81,7 @@ def inherit(files, fn, lineno, d, deferred=False):
            __inherit_cache = d.getVar('__inherit_cache', False) or []
 def get_statements(filename, absolute_filename, base_name):
-    global cached_statements
+    global cached_statements, __residue__, __body__
    try:
        return cached_statements[absolute_filename]
@@ -100,6 +101,11 @@ def get_statements(filename, absolute_filename, base_name):
            # add a blank line to close out any python definition
            feeder(lineno, "", filename, base_name, statements, eof=True)
+        if __residue__:
+            raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno)
+        if __body__:
+            raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno)
        if filename.endswith(".bbclass") or filename.endswith(".inc"):
            cached_statements[absolute_filename] = statements
        return statements
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index bc7e18175d..3462ed4457 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1273,27 +1273,41 @@ class RunQueueData:
        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
+        starttime = time.time()
+        lasttime = starttime
        # Iterate over the task list and call into the siggen code
        dealtwith = set()
        todeal = set(self.runtaskentries)
        while todeal:
+            ready = set()
            for tid in todeal.copy():
                if not (self.runtaskentries[tid].depends - dealtwith):
-                    dealtwith.add(tid)
+                    self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-                    todeal.remove(tid)
+                    # get_taskhash for a given tid *must* be called before get_unihash* below
-                    self.prepare_task_hash(tid)
+                    self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-                bb.event.check_for_interrupts(self.cooker.data)
+                    ready.add(tid)
+            unihashes = bb.parse.siggen.get_unihashes(ready)
+            for tid in ready:
+                dealtwith.add(tid)
+                todeal.remove(tid)
+                self.runtaskentries[tid].unihash = unihashes[tid]
+            bb.event.check_for_interrupts(self.cooker.data)
+            if time.time() > (lasttime + 30):
+                lasttime = time.time()
+                hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime))
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime))
        bb.parse.siggen.writeout_file_checksum_cache()
        #self.dump_data()
        return len(self.runtaskentries)
-    def prepare_task_hash(self, tid):
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
    def dump_data(self):
        """
        Dump some debug information on the internal data structures
@@ -2438,14 +2452,17 @@ class RunQueueExecute:
        taskdepdata_cache = {}
        for task in self.rqdata.runtaskentries:
            (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
-            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
+            taskdepdata_cache[task] = bb.TaskData(
-            deps = self.rqdata.runtaskentries[task].depends
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
-            provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
+                taskname = taskname,
-            taskhash = self.rqdata.runtaskentries[task].hash
+                fn = fn,
-            unihash = self.rqdata.runtaskentries[task].unihash
+                deps = self.filtermcdeps(task, mc, self.rqdata.runtaskentries[task].depends),
-            deps = self.filtermcdeps(task, mc, deps)
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
-            hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
+                taskhash = self.rqdata.runtaskentries[task].hash,
-            taskdepdata_cache[task] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+                unihash = self.rqdata.runtaskentries[task].unihash,
+                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                taskhash_deps = self.rqdata.runtaskentries[task].taskhash_deps,
+            )
        self.taskdepdata_cache = taskdepdata_cache
@@ -2460,9 +2477,11 @@ class RunQueueExecute:
        while next:
            additional = []
            for revdep in next:
-                self.taskdepdata_cache[revdep][6] = self.rqdata.runtaskentries[revdep].unihash
+                self.taskdepdata_cache[revdep] = self.taskdepdata_cache[revdep]._replace(
+                    unihash=self.rqdata.runtaskentries[revdep].unihash
+                )
                taskdepdata[revdep] = self.taskdepdata_cache[revdep]
-                for revdep2 in self.taskdepdata_cache[revdep][3]:
+                for revdep2 in self.taskdepdata_cache[revdep].deps:
                    if revdep2 not in taskdepdata:
                        additional.append(revdep2)
            next = additional
@@ -2531,9 +2550,6 @@ class RunQueueExecute:
                    self.rqdata.runtaskentries[hashtid].unihash = unihash
                    bb.parse.siggen.set_unihash(hashtid, unihash)
                    toprocess.add(hashtid)
-                if torehash:
-                    # Need to save after set_unihash above
-                    bb.parse.siggen.save_unitaskhashes()
        # Work out all tasks which depend upon these
        total = set()
@@ -2556,17 +2572,28 @@ class RunQueueExecute:
            elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
                next.add(p)
+        starttime = time.time()
+        lasttime = starttime
        # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
        while next:
            current = next.copy()
            next = set()
+            ready = {}
            for tid in current:
                if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                    continue
+                # get_taskhash for a given tid *must* be called before get_unihash* below
+                ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+            unihashes = bb.parse.siggen.get_unihashes(ready.keys())
+            for tid in ready:
                orighash = self.rqdata.runtaskentries[tid].hash
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+                newhash = ready[tid]
                origuni = self.rqdata.runtaskentries[tid].unihash
-                newuni = bb.parse.siggen.get_unihash(tid)
+                newuni = unihashes[tid]
                # FIXME, need to check it can come from sstate at all for determinism?
                remapped = False
                if newuni == origuni:
@@ -2587,6 +2614,15 @@ class RunQueueExecute:
                next |= self.rqdata.runtaskentries[tid].revdeps
                total.remove(tid)
                next.intersection_update(total)
+                bb.event.check_for_interrupts(self.cooker.data)
+                if time.time() > (lasttime + 30):
+                    lasttime = time.time()
+                    hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime))
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime))
        if changed:
            for mc in self.rq.worker:
@@ -2806,13 +2842,19 @@ class RunQueueExecute:
            additional = []
            for revdep in next:
                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                deps = getsetscenedeps(revdep)
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
+                taskdepdata[revdep] = bb.TaskData(
-                unihash = self.rqdata.runtaskentries[revdep].unihash
+                    pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
-                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
+                    taskname = taskname,
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+                    fn = fn,
+                    deps = deps,
+                    provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                    taskhash = self.rqdata.runtaskentries[revdep].hash,
+                    unihash = self.rqdata.runtaskentries[revdep].unihash,
+                    hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                    taskhash_deps = self.rqdata.runtaskentries[revdep].taskhash_deps,
+                )
                for revdep2 in deps:
                    if revdep2 not in taskdepdata:
                        additional.append(revdep2)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 2a0ecf57e1..89b70fb6a4 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -15,6 +15,7 @@ import difflib
 import simplediff
 import json
 import types
+from contextlib import contextmanager
 import bb.compress.zstd
 from bb.checksum import FileChecksumCache
 from bb import runqueue
@@ -28,6 +29,14 @@ hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 # The minimum version of the find_siginfo function we need
 find_siginfo_minversion = 2
+HASHSERV_ENVVARS = [
+    "SSL_CERT_DIR",
+    "SSL_CERT_FILE",
+    "NO_PROXY",
+    "HTTPS_PROXY",
+    "HTTP_PROXY"
+]
 def check_siggen_version(siggen):
    if not hasattr(siggen, "find_siginfo_version"):
        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
@@ -192,9 +201,6 @@ class SignatureGenerator(object):
    def save_unitaskhashes(self):
        return
-    def copy_unitaskhashes(self, targetdir):
-        return
    def set_setscene_tasks(self, setscene_tasks):
        return
@@ -372,7 +378,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
            self.taints[tid] = taint
            logger.warning("%s is tainted from a forced run" % tid)
-        return
+        return set(dep for _, dep in self.runtaskdeps[tid])
    def get_taskhash(self, tid, deps, dataCaches):
@@ -409,9 +415,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
    def save_unitaskhashes(self):
        self.unihash_cache.save(self.unitaskhashes)
-    def copy_unitaskhashes(self, targetdir):
-        self.unihash_cache.copyfile(targetdir)
    def dump_sigtask(self, mcfn, task, stampbase, runtime):
        tid = mcfn + ":" + task
        mc = bb.runqueue.mc_from_tid(mcfn)
@@ -531,19 +534,28 @@ class SignatureGeneratorUniHashMixIn(object):
    def __init__(self, data):
        self.extramethod = {}
        # NOTE: The cache only tracks hashes that exist. Hashes that don't
-        # exist are always queries from the server since it is possible for
+        # exist are always queried from the server since it is possible for
        # hashes to appear over time, but much less likely for them to
        # disappear
        self.unihash_exists_cache = set()
        self.username = None
        self.password = None
+        self.env = {}
+        origenv = data.getVar("BB_ORIGENV")
+        for e in HASHSERV_ENVVARS:
+            value = data.getVar(e)
+            if not value and origenv:
+                value = origenv.getVar(e)
+            if value:
+                self.env[e] = value
        super().__init__(data)
    def get_taskdata(self):
-        return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password) + super().get_taskdata()
+        return (self.server, self.method, self.extramethod, self.username, self.password, self.env) + super().get_taskdata()
    def set_taskdata(self, data):
-        self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password = data[:6]
+        self.server, self.method, self.extramethod, self.username, self.password, self.env = data[:6]
        super().set_taskdata(data[6:])
    def get_hashserv_creds(self):
@@ -555,15 +567,27 @@ class SignatureGeneratorUniHashMixIn(object):
        return {}
-    def client(self):
+    @contextmanager
-        if getattr(self, '_client', None) is None:
+    def _client_env(self):
-            self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+        orig_env = os.environ.copy()
-        return self._client
+        try:
+            for k, v in self.env.items():
+                os.environ[k] = v
+            yield
+        finally:
+            for k, v in self.env.items():
+                if k in orig_env:
+                    os.environ[k] = orig_env[k]
+                else:
+                    del os.environ[k]
-    def client_pool(self):
+    @contextmanager
-        if getattr(self, '_client_pool', None) is None:
+    def client(self):
-            self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
+        with self._client_env():
-        return self._client_pool
+            if getattr(self, '_client', None) is None:
+                self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+            yield self._client
    def reset(self, data):
        self.__close_clients()
@@ -574,12 +598,13 @@ class SignatureGeneratorUniHashMixIn(object):
        return super().exit()
    def __close_clients(self):
-        if getattr(self, '_client', None) is not None:
+        with self._client_env():
-            self._client.close()
+            if getattr(self, '_client', None) is not None:
-            self._client = None
+                self._client.close()
-        if getattr(self, '_client_pool', None) is not None:
+                self._client = None
-            self._client_pool.close()
+            if getattr(self, '_client_pool', None) is not None:
-            self._client_pool = None
+                self._client_pool.close()
+                self._client_pool = None
    def get_stampfile_hash(self, tid):
        if tid in self.taskhash:
@@ -640,23 +665,20 @@ class SignatureGeneratorUniHashMixIn(object):
        if len(query) == 0:
            return {}
-        uncached_query = {}
+        query_keys = []
        result = {}
        for key, unihash in query.items():
            if unihash in self.unihash_exists_cache:
                result[key] = True
            else:
-                uncached_query[key] = unihash
+                query_keys.append(key)
-        if self.max_parallel <= 1 or len(uncached_query) <= 1:
+        if query_keys:
-            # No parallelism required. Make the query serially with the single client
+            with self.client() as client:
-            uncached_result = {
+                query_result = client.unihash_exists_batch(query[k] for k in query_keys)
-                key: self.client().unihash_exists(value) for key, value in uncached_query.items()
-            }
-        else:
-            uncached_result = self.client_pool().unihashes_exist(uncached_query)
-        for key, exists in uncached_result.items():
+        for idx, key in enumerate(query_keys):
+            exists = query_result[idx]
            if exists:
                self.unihash_exists_cache.add(query[key])
            result[key] = exists
@@ -672,27 +694,20 @@ class SignatureGeneratorUniHashMixIn(object):
        unihash
        """
        result = {}
-        queries = {}
+        query_tids = []
-        query_result = {}
        for tid in tids:
            unihash = self.get_cached_unihash(tid)
            if unihash:
                result[tid] = unihash
            else:
-                queries[tid] = (self._get_method(tid), self.taskhash[tid])
+                query_tids.append(tid)
-        if len(queries) == 0:
+        if query_tids:
-            return result
+            with self.client() as client:
+                unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids)
-        if self.max_parallel <= 1 or len(queries) <= 1:
+        for idx, tid in enumerate(query_tids):
-            # No parallelism required. Make the query serially with the single client
-            for tid, args in queries.items():
-                query_result[tid] = self.client().get_unihash(*args)
-        else:
-            query_result = self.client_pool().get_unihashes(queries)
-        for tid, unihash in query_result.items():
            # In the absence of being able to discover a unique hash from the
            # server, make it be equivalent to the taskhash. The unique "hash" only
            # really needs to be a unique string (not even necessarily a hash), but
@@ -707,6 +722,8 @@ class SignatureGeneratorUniHashMixIn(object):
            #    to the server, there is a better chance that they will agree on
            #    the unique hash.
            taskhash = self.taskhash[tid]
+            unihash = unihashes[idx]
            if unihash:
                # A unique hash equal to the taskhash is not very interesting,
                # so it is reported it at debug level 2. If they differ, that
@@ -785,7 +802,9 @@ class SignatureGeneratorUniHashMixIn(object):
                if tid in self.extramethod:
                    method = method + self.extramethod[tid]
-                data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
+                with self.client() as client:
+                    data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
                new_unihash = data['unihash']
                if new_unihash != unihash:
@@ -816,7 +835,9 @@ class SignatureGeneratorUniHashMixIn(object):
            if tid in self.extramethod:
                method = method + self.extramethod[tid]
-            data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+            with self.client() as client:
+                data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
            hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
            if data is None:
@@ -849,7 +870,6 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
        super().init_rundepcheck(data)
        self.server = data.getVar('BB_HASHSERVE')
        self.method = "sstate_output_hash"
-        self.max_parallel = 1
 def clean_checksum_file_path(file_checksum_tuple):
    f, cs = file_checksum_tuple
diff --git a/bitbake/lib/bb/tests/codeparser.py b/bitbake/lib/bb/tests/codeparser.py
index f6585fb3aa..c0d1362a0c 100644
--- a/bitbake/lib/bb/tests/codeparser.py
+++ b/bitbake/lib/bb/tests/codeparser.py
@@ -106,6 +106,46 @@ ${D}${libdir}/pkgconfig/*.pc
        self.parseExpression("foo=$(echo bar)")
        self.assertExecs(set(["echo"]))
+    def test_assign_subshell_expansion_quotes(self):
+        self.parseExpression('foo="$(echo bar)"')
+        self.assertExecs(set(["echo"]))
+    def test_assign_subshell_expansion_nested(self):
+        self.parseExpression('foo="$(func1 "$(func2 bar$(func3))")"')
+        self.assertExecs(set(["func1", "func2", "func3"]))
+    def test_assign_subshell_expansion_multiple(self):
+        self.parseExpression('foo="$(func1 "$(func2)") $(func3)"')
+        self.assertExecs(set(["func1", "func2", "func3"]))
+    def test_assign_subshell_expansion_escaped_quotes(self):
+        self.parseExpression('foo="\\"fo\\"o$(func1)"')
+        self.assertExecs(set(["func1"]))
+    def test_assign_subshell_expansion_empty(self):
+        self.parseExpression('foo="bar$()foo"')
+        self.assertExecs(set())
+    def test_assign_subshell_backticks(self):
+        self.parseExpression("foo=`echo bar`")
+        self.assertExecs(set(["echo"]))
+    def test_assign_subshell_backticks_quotes(self):
+        self.parseExpression('foo="`echo bar`"')
+        self.assertExecs(set(["echo"]))
+    def test_assign_subshell_backticks_multiple(self):
+        self.parseExpression('foo="`func1 bar` `func2`"')
+        self.assertExecs(set(["func1", "func2"]))
+    def test_assign_subshell_backticks_escaped_quotes(self):
+        self.parseExpression('foo="\\"fo\\"o`func1`"')
+        self.assertExecs(set(["func1"]))
+    def test_assign_subshell_backticks_empty(self):
+        self.parseExpression('foo="bar``foo"')
+        self.assertExecs(set())
    def test_shell_unexpanded(self):
        self.setEmptyVars(["QT_BASE_NAME"])
        self.parseExpression('echo "${QT_BASE_NAME}"')
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 85c1f79ff3..701129d138 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -511,7 +511,8 @@ class MirrorUriTest(FetcherTest):
    mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
                "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
                "https://.*/.* file:///someotherpath/downloads/ " \
-                "http://.*/.* file:///someotherpath/downloads/"
+                "http://.*/.* file:///someotherpath/downloads/ " \
+                "svn://svn.server1.com/ svn://svn.server2.com/"
    def test_urireplace(self):
        self.d.setVar("FILESPATH", ".")
@@ -535,6 +536,13 @@ class MirrorUriTest(FetcherTest):
        uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
        self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
+    def test_urilistsvn(self):
+        # Catch svn:// -> svn:// bug
+        fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d)
+        mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
+        uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
+        self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2'])
    def test_mirror_of_mirror(self):
        # Test if mirror of a mirror works
        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
@@ -1421,7 +1429,7 @@ class FetchLatestVersionTest(FetcherTest):
        # combination version pattern
        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
            : "1.2.0",
-        ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
+        ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
            : "2014.01",
        # version pattern "yyyymmdd"
        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
@@ -1493,6 +1501,12 @@ class FetchLatestVersionTest(FetcherTest):
            : "2.8",
    }
+    test_crate_uris = {
+        # basic example; version pattern "A.B.C+cargo-D.E.F"
+        ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69")
+            : "0.9.29"
+   }
    @skipIfNoNetwork()
    def test_git_latest_versionstring(self):
        for k, v in self.test_git_uris.items():
@@ -1511,7 +1525,7 @@ class FetchLatestVersionTest(FetcherTest):
    def test_wget_latest_versionstring(self):
        testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
-        server = HTTPService(testdata)
+        server = HTTPService(testdata, host="127.0.0.1")
        server.start()
        port = server.port
        try:
@@ -1519,10 +1533,10 @@ class FetchLatestVersionTest(FetcherTest):
                self.d.setVar("PN", k[0])
                checkuri = ""
                if k[2]:
-                    checkuri = "http://localhost:%s/" % port + k[2]
+                    checkuri = "http://127.0.0.1:%s/" % port + k[2]
                self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
                self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
-                url = "http://localhost:%s/" % port + k[1]
+                url = "http://127.0.0.1:%s/" % port + k[1]
                ud = bb.fetch2.FetchData(url, self.d)
                pupver = ud.method.latest_versionstring(ud, self.d)
                verstring = pupver[0]
@@ -1532,6 +1546,16 @@ class FetchLatestVersionTest(FetcherTest):
        finally:
            server.stop()
+    @skipIfNoNetwork()
+    def test_crate_latest_versionstring(self):
+        for k, v in self.test_crate_uris.items():
+            self.d.setVar("PN", k[0])
+            ud = bb.fetch2.FetchData(k[1], self.d)
+            pupver = ud.method.latest_versionstring(ud, self.d)
+            verstring = pupver[0]
+            self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
+            r = bb.utils.vercmp_string(v, verstring)
+            self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 class FetchCheckStatusTest(FetcherTest):
    test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 8b212b7803..4ee45d67a2 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -559,7 +559,10 @@ class ORMWrapper(object):
                # we might have an invalid link; no way to detect this. just set it to None
                filetarget_obj = None
-            parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
+            try:
+                parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
+            except Target_File.DoesNotExist:
+                parent_obj = None
            Target_File.objects.create(
                        target = target_obj,